From df4a28b8db98613905932a608c5e0898587c8187 Mon Sep 17 00:00:00 2001 From: Kritka Sahni Date: Tue, 26 Sep 2023 12:09:46 -0700 Subject: [PATCH] Added support for vendored pip dependencies in python buildpack via build envvar GOOGLE_VENDOR_PIP_DEPENDENCIES. PiperOrigin-RevId: 568612289 Change-Id: Id6871f3b30a13ae1a7a5e2fcb15b23a9fac4f32f --- builders/python/acceptance/gcp_test.go | 6 ++ .../pip_vendored_dependencies/Procfile | 1 + .../generic/pip_vendored_dependencies/main.py | 52 ++++++++++++++++++ ...mple_pip_dependency-0.0.1-py3-none-any.whl | Bin 0 -> 1810 bytes .../python-package/README.md | 10 ++++ .../python-package/pyproject.toml | 22 ++++++++ .../src/sample_pip_dependency/__init__.py | 0 .../src/sample_pip_dependency/sample.py | 2 + .../python-package/tests/README.md | 4 ++ .../requirements.txt | 1 + pkg/buildermetrics/counterids.go | 5 ++ pkg/python/BUILD.bazel | 1 + pkg/python/python.go | 9 +++ 13 files changed, 113 insertions(+) create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/Procfile create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/main.py create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/package/sample_pip_dependency-0.0.1-py3-none-any.whl create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/python-package/README.md create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/python-package/pyproject.toml create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/__init__.py create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/sample.py create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/python-package/tests/README.md create mode 100644 builders/testdata/python/generic/pip_vendored_dependencies/requirements.txt diff --git a/builders/python/acceptance/gcp_test.go b/builders/python/acceptance/gcp_test.go index 367c8b1c0..ea1aa0e6e 100644 --- a/builders/python/acceptance/gcp_test.go +++ b/builders/python/acceptance/gcp_test.go @@ -76,6 +76,12 @@ func TestAcceptancePython(t *testing.T) { // numpy requires Python 3.8 or newer. VersionInclusionConstraint: ">= 3.8.0", }, + { + Name: "pip vendored dependencies", + App: "pip_vendored_dependencies", + Env: []string{"GOOGLE_VENDOR_PIP_DEPENDENCIES=package"}, + MustUse: []string{pythonRuntime, pythonPIP, entrypoint}, + }, } for _, tc := range acceptance.FilterTests(t, imageCtx, testCases) { diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/Procfile b/builders/testdata/python/generic/pip_vendored_dependencies/Procfile new file mode 100644 index 000000000..eb399cbcf --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/Procfile @@ -0,0 +1 @@ +web: python3 main.py \ No newline at end of file diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/main.py b/builders/testdata/python/generic/pip_vendored_dependencies/main.py new file mode 100644 index 000000000..7d25a22e5 --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/main.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Simple web server used to validate that pip vendored deps are installed. +""" +from http.server import BaseHTTPRequestHandler, HTTPServer +from sample_pip_dependency import sample + + +""" Simple web server to respond to HTTP GET. +""" +class MyServer(BaseHTTPRequestHandler): + def _set_headers(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def _html(self, message): + """This just generates an HTML document that includes `message` + in the body. + """ + content = f"{message}" + return content.encode("utf8") # NOTE: must return a bytes object! + + def do_GET(self): + self._set_headers() + message = sample.helloworld() + if message == "hello world - from pip dependency": + self.wfile.write(self._html("PASS")) + else: + self.wfile.write(self._html("FAIL")) + +if __name__ == "__main__": + webServer = HTTPServer(("0.0.0.0", 8080), MyServer) + + try: + webServer.serve_forever() + except KeyboardInterrupt: + pass + + webServer.server_close() diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/package/sample_pip_dependency-0.0.1-py3-none-any.whl b/builders/testdata/python/generic/pip_vendored_dependencies/package/sample_pip_dependency-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..1e18245321e2a6fda38e8d08d3e13bf6ce5c12de GIT binary patch literal 1810 zcmWIWW@Zs#U|`^2D4k;&4rDMefe0W_1mfbv+=86c_=3!W_>|Ow)V!3`yyQy#`1s7c z%#!$cy@E<+h5!@;I=Y3|SpzjX0g`H}o(?csYhEuGO#)U#PIzK~Z(9f~mf$ss7HOPHrT7mjE5^qrm+miIIUJ z80dOET=wc3=o#o4>ZN2Bm*{5ZrRD4Ux`sHqIEFa(2AaEB2ZjdG8(Y{TTT6 z;zUL_arR3W-R=gd+@F-M_Eo+jHYPp8)~Y?zXahr;1&_mebt#vez8iub42ohWj6O_q zby_60NpAH~jarXcGPe#M=udgt5s>^|x5jRs+PwDlu^e|DT!rtSRY4)Z!YrZ^wFBx@AGpVvq;QSKrPun*(JD=)b^2M(%bEbRJR+$Mm?IOL{ zHl<9zVENkm#-jJfEH|Be9(?ZE^Sf_Z>$0Uk)im9?E*)`X&EHA(D(82fefaiQQ<1#p z!ZU1Nk2t+M)7o3F?RtgzuVk*$we)D_^7y=N|18%#D$=Ugmrg(E-S+CRNby#u^+C_> z_2vF}J@psE*QD^=>$`4+RBc^%?f2EH{ePBTS-732QgK6V{OZrF zu>``}M8{URhpVd(_jwQBvln^2b+yi&Ilno`;EM5sBLB1Ax?Vab^*051c%9VI>(pmo z8F<0qg0bNh<4aGtPMz1k_*Lf`uZFJI*^@q-f;2R>e4lc8`fLxYvc*@b}d9IP2|s z)_dI(bcKggx`?Czo_2L^|W_wpH4PJGe`QqWOitmG1& zE$edN>C-eu;UB+5B-4W;Zrpn`Pw}-!v_cc-E|DcK^j+tl&u%H`cW<#UGCpi7-0(&3 z6dRAn+KAg4$$@*`2Z`ONGGukVQS#t{khSewv+Ef~{XF}n=H>l=wg073$-`}5zjkFE zEBdaxSXlg|kBzX`Ea^9XhTB*+=$sUBbYnWdUg!MBJJRI`51)9nZ_nCTiR?D9Y<`u+ zW_RD%%UDlN$X)%mT;jcfo}AKBzd74D zony|FJf*3dM80n0vq_#aX}!>NQ>|jRicqeF?^@Xouc>1U@MdHZVa8qJ0mB~*7Bqq= zg5@8&ap>hA#4rYi1&yE*5xa5l(h%7&Y=s)au${OJ!&$_kn~k0Y5oV_YlOa}zVweri ykmv@ZXB&jUJj}!zjFO4aEkMsMFblRcmJ)3NB=ZD#v$BD7vjHIw(AOVWKs*2`frjq@ literal 0 HcmV?d00001 diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/python-package/README.md b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/README.md new file mode 100644 index 000000000..8d5ce066d --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/README.md @@ -0,0 +1,10 @@ +# SAMPLE-PIP-DEPENDENCY + +This is a simple package to test pip vendored deps. + +# Generate distribution package +Use "python3 -m build" in python-package to generate .whl binary. pyproject.toml has all details to create distribution package. +Leave tests/ and __init__.py empty - these are merely required to generate distribtuion package. + +# Usage +Import this as "from sample_pip_dependency import sample" and use as sample.helloworld() to test. diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/python-package/pyproject.toml b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/pyproject.toml new file mode 100644 index 000000000..99d3cb9cd --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "sample_pip_dependency" +version = "0.0.1" +authors = [ + { name="Kritka Sahni", email="kritkasahni@google.com" }, +] +description = "A small example package to test pip vendored deps" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://github.com/pypa/sampleproject" +"Bug Tracker" = "https://github.com/pypa/sampleproject/issues" diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/__init__.py b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/sample.py b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/sample.py new file mode 100644 index 000000000..798e658c8 --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/sample.py @@ -0,0 +1,2 @@ +def helloworld(): + return "hello world - from pip dependency" diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/python-package/tests/README.md b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/tests/README.md new file mode 100644 index 000000000..284e3fefe --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/python-package/tests/README.md @@ -0,0 +1,4 @@ +# SAMPLE-PIP-DEPENDENCY +# Generate distribution package +Leave tests/ empty - it is merely required to generate distribution package. + diff --git a/builders/testdata/python/generic/pip_vendored_dependencies/requirements.txt b/builders/testdata/python/generic/pip_vendored_dependencies/requirements.txt new file mode 100644 index 000000000..901f1e25d --- /dev/null +++ b/builders/testdata/python/generic/pip_vendored_dependencies/requirements.txt @@ -0,0 +1 @@ +sample-pip-dependency==0.0.1 \ No newline at end of file diff --git a/pkg/buildermetrics/counterids.go b/pkg/buildermetrics/counterids.go index c30846e0c..c442076df 100644 --- a/pkg/buildermetrics/counterids.go +++ b/pkg/buildermetrics/counterids.go @@ -38,6 +38,7 @@ const ( NpmGcpBuildUsageCounterID CounterID = "2" NpmBuildUsageCounterID CounterID = "3" NpmGoogleNodeRunScriptsUsageCounterID CounterID = "4" + PipVendorDependenciesCounterID CounterID = "5" ) var ( @@ -58,6 +59,10 @@ var ( "npm_google_node_run_script_uses", "The number of times the GOOGLE_NODE_RUN_SCRIPTS env var is used by npm developers", }, + PipVendorDependenciesCounterID: Descriptor{ + "vendor_pip_dependencies_uses", + "The number of times GOOGLE_VENDOR_PIP_DEPENDENCIES is used by developers", + }, } ) diff --git a/pkg/python/BUILD.bazel b/pkg/python/BUILD.bazel index 9bce2f3ed..d8f96cd78 100644 --- a/pkg/python/BUILD.bazel +++ b/pkg/python/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//cmd/python:__subpackages__"], deps = [ "//pkg/ar", + "//pkg/buildermetrics", "//pkg/cache", "//pkg/env", "//pkg/gcpbuildpack", diff --git a/pkg/python/python.go b/pkg/python/python.go index 2b800b3ea..ebd5eafff 100644 --- a/pkg/python/python.go +++ b/pkg/python/python.go @@ -23,6 +23,7 @@ import ( "time" "github.com/GoogleCloudPlatform/buildpacks/pkg/ar" + "github.com/GoogleCloudPlatform/buildpacks/pkg/buildermetrics" "github.com/GoogleCloudPlatform/buildpacks/pkg/cache" "github.com/GoogleCloudPlatform/buildpacks/pkg/env" gcp "github.com/GoogleCloudPlatform/buildpacks/pkg/gcpbuildpack" @@ -44,6 +45,9 @@ const ( // The requirements files are processed from left to right, with requirements from the next overriding any conflicts from the previous. RequirementsFilesEnv = "GOOGLE_INTERNAL_REQUIREMENTS_FILES" + // VendorPipDepsEnv is the envar used to opt using vendored pip dependencies + VendorPipDepsEnv = "GOOGLE_VENDOR_PIP_DEPENDENCIES" + versionFile = ".python-version" versionKey = "version" versionEnv = "GOOGLE_PYTHON_VERSION" @@ -235,6 +239,11 @@ func InstallRequirements(ctx *gcp.Context, l *libcnb.Layer, reqs ...string) erro "--disable-pip-version-check", // If we were going to upgrade pip, we would have done it already in the runtime buildpack. "--no-cache-dir", // We used to save this to a layer, but it made builds slower because it includes http caching of pypi requests. } + vendorDir, isVendored := os.LookupEnv(VendorPipDepsEnv) + if isVendored { + cmd = append(cmd, "--no-index", "--find-links", vendorDir) + buildermetrics.GlobalBuilderMetrics().GetCounter(buildermetrics.PipVendorDependenciesCounterID).Increment(1) + } if !virtualEnv { cmd = append(cmd, "--user") // Install into user site-packages directory. }