Added support for vendored pip dependencies in python buildpack via b…

…uild envvar GOOGLE_VENDOR_PIP_DEPENDENCIES.

PiperOrigin-RevId: 568612289
Change-Id: Id6871f3b30a13ae1a7a5e2fcb15b23a9fac4f32f

builders/python/acceptance/gcp_test.go

@@ -76,6 +76,12 @@ func TestAcceptancePython(t *testing.T) {
 			// numpy requires Python 3.8 or newer.
 			VersionInclusionConstraint: ">= 3.8.0",
 		},
+		{
+			Name:    "pip vendored dependencies",
+			App:     "pip_vendored_dependencies",
+			Env:     []string{"GOOGLE_VENDOR_PIP_DEPENDENCIES=package"},
+			MustUse: []string{pythonRuntime, pythonPIP, entrypoint},
+		},
 	}
 
 	for _, tc := range acceptance.FilterTests(t, imageCtx, testCases) {

builders/testdata/python/generic/pip_vendored_dependencies/Procfile

@@ -0,0 +1 @@
+web: python3 main.py

builders/testdata/python/generic/pip_vendored_dependencies/main.py

@@ -0,0 +1,52 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple web server used to validate that pip vendored deps are installed.
+"""
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from sample_pip_dependency import sample
+
+
+""" Simple web server to respond to HTTP GET.
+"""
+class MyServer(BaseHTTPRequestHandler):
+    def _set_headers(self):
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+
+    def _html(self, message):
+        """This just generates an HTML document that includes `message`
+        in the body.
+        """
+        content = f"{message}"
+        return content.encode("utf8")  # NOTE: must return a bytes object!
+
+    def do_GET(self):
+        self._set_headers()
+        message = sample.helloworld()
+        if message == "hello world - from pip dependency":
+          self.wfile.write(self._html("PASS"))
+        else:
+          self.wfile.write(self._html("FAIL"))
+
+if __name__ == "__main__":
+    webServer = HTTPServer(("0.0.0.0", 8080), MyServer)
+
+    try:
+        webServer.serve_forever()
+    except KeyboardInterrupt:
+        pass
+
+    webServer.server_close()

builders/testdata/python/generic/pip_vendored_dependencies/python-package/README.md

@@ -0,0 +1,10 @@
+# SAMPLE-PIP-DEPENDENCY
+
+This is a simple package to test pip vendored deps.
+
+# Generate distribution package
+Use "python3 -m build" in python-package to generate .whl binary. pyproject.toml has all details to create distribution package.
+Leave tests/ and __init__.py empty - these are merely required to generate distribtuion package.
+
+# Usage
+Import this as "from sample_pip_dependency import sample" and use as sample.helloworld() to test.

builders/testdata/python/generic/pip_vendored_dependencies/python-package/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sample_pip_dependency"
+version = "0.0.1"
+authors = [
+  { name="Kritka Sahni", email="[email protected]" },
+]
+description = "A small example package to test pip vendored deps"
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/pypa/sampleproject"
+"Bug Tracker" = "https://github.com/pypa/sampleproject/issues"

builders/testdata/python/generic/pip_vendored_dependencies/python-package/src/sample_pip_dependency/sample.py

@@ -0,0 +1,2 @@
+def helloworld():
+    return "hello world - from pip dependency"

builders/testdata/python/generic/pip_vendored_dependencies/python-package/tests/README.md

@@ -0,0 +1,4 @@
+# SAMPLE-PIP-DEPENDENCY
+# Generate distribution package
+Leave tests/ empty - it is merely required to generate distribution package.
+

builders/testdata/python/generic/pip_vendored_dependencies/requirements.txt

@@ -0,0 +1 @@
+sample-pip-dependency==0.0.1

pkg/buildermetrics/counterids.go

@@ -38,6 +38,7 @@ const (
 	NpmGcpBuildUsageCounterID             CounterID = "2"
 	NpmBuildUsageCounterID                CounterID = "3"
 	NpmGoogleNodeRunScriptsUsageCounterID CounterID = "4"
+	PipVendorDependenciesCounterID        CounterID = "5"
 )
 
 var (
@@ -58,6 +59,10 @@ var (
 			"npm_google_node_run_script_uses",
 			"The number of times the GOOGLE_NODE_RUN_SCRIPTS env var is used by npm developers",
 		},
+		PipVendorDependenciesCounterID: Descriptor{
+			"vendor_pip_dependencies_uses",
+			"The number of times GOOGLE_VENDOR_PIP_DEPENDENCIES is used by developers",
+		},
 	}
 )
 

pkg/python/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//cmd/python:__subpackages__"],
     deps = [
         "//pkg/ar",
+        "//pkg/buildermetrics",
         "//pkg/cache",
         "//pkg/env",
         "//pkg/gcpbuildpack",

pkg/python/python.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/GoogleCloudPlatform/buildpacks/pkg/ar"
+	"github.com/GoogleCloudPlatform/buildpacks/pkg/buildermetrics"
 	"github.com/GoogleCloudPlatform/buildpacks/pkg/cache"
 	"github.com/GoogleCloudPlatform/buildpacks/pkg/env"
 	gcp "github.com/GoogleCloudPlatform/buildpacks/pkg/gcpbuildpack"
@@ -44,6 +45,9 @@ const (
 	// The requirements files are processed from left to right, with requirements from the next overriding any conflicts from the previous.
 	RequirementsFilesEnv = "GOOGLE_INTERNAL_REQUIREMENTS_FILES"
 
+	// VendorPipDepsEnv is the envar used to opt using vendored pip dependencies
+	VendorPipDepsEnv = "GOOGLE_VENDOR_PIP_DEPENDENCIES"
+
 	versionFile = ".python-version"
 	versionKey  = "version"
 	versionEnv  = "GOOGLE_PYTHON_VERSION"
@@ -235,6 +239,11 @@ func InstallRequirements(ctx *gcp.Context, l *libcnb.Layer, reqs ...string) erro
 			"--disable-pip-version-check", // If we were going to upgrade pip, we would have done it already in the runtime buildpack.
 			"--no-cache-dir",              // We used to save this to a layer, but it made builds slower because it includes http caching of pypi requests.
 		}
+		vendorDir, isVendored := os.LookupEnv(VendorPipDepsEnv)
+		if isVendored {
+			cmd = append(cmd, "--no-index", "--find-links", vendorDir)
+			buildermetrics.GlobalBuilderMetrics().GetCounter(buildermetrics.PipVendorDependenciesCounterID).Increment(1)
+		}
 		if !virtualEnv {
 			cmd = append(cmd, "--user") // Install into user site-packages directory.
 		}