Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Add tracing with OpenTelemetry #932

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open

[WIP] Add tracing with OpenTelemetry #932

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
1ad8b12
install Go otel packages
meobilivang May 4, 2023
f955a09
set up TracerProvider + tracer object
meobilivang May 4, 2023
dcfb15b
add enableTracing flag
meobilivang May 4, 2023
49c1ea2
instrument GCP machine + GCPCluster controllers
meobilivang May 4, 2023
deb44dc
helm charts for jaeger all-in-one + otel collector
meobilivang May 5, 2023
bb98f0e
templates to set up dev env
meobilivang May 5, 2023
bc4ea9f
instrument cloud/scope
meobilivang May 5, 2023
eb25369
instrument exp/controllers
meobilivang May 7, 2023
1b98f8e
instrument cloud/services/compute
meobilivang May 7, 2023
ea54fe7
instrument cloud/services/container
meobilivang May 7, 2023
560fd0a
add sampling rate
meobilivang May 12, 2023
ba6261e
Tilt file
meobilivang May 12, 2023
173a97f
debug blocking tracing connection
meobilivang May 22, 2023
f4a0554
helm charts for jaeger + otel collector
meobilivang May 26, 2023
fc2812d
bump go package
meobilivang May 26, 2023
e4da793
bump packages
meobilivang May 26, 2023
085b48f
Merge branch 'main' into add-tracing-otel
meobilivang May 26, 2023
bbc5bf5
reformat tilt file
meobilivang Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 179 additions & 46 deletions Tiltfile
View file
Open in desktop
Copy link
Contributor Author

@meobilivang meobilivang May 26, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My python environment has the Black linter by default so it just reformats the whole file 😗

Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ tools_bin = "./hack/tools/bin"
kubectl_cmd = "./hack/tools/bin/kubectl"
kind_cmd = "./hack/tools/bin/kind"

#Add tools to path
# Add tools to path
os.putenv("PATH", os.getenv("PATH") + ":" + tools_bin)

update_settings(k8s_upsert_timeout_secs = 60) # on first tilt up, often can take longer than 30 seconds
update_settings(
k8s_upsert_timeout_secs=60
) # on first tilt up, often can take longer than 30 seconds

# set defaults
settings = {
Expand All @@ -26,10 +28,12 @@ settings = {
keys = ["GCP_B64ENCODED_CREDENTIALS"]

# global settings
settings.update(read_json(
"tilt-settings.json",
default = {},
))
settings.update(
read_json(
"tilt-settings.json",
default={},
)
)

if settings.get("trigger_mode") == "manual":
trigger_mode(TRIGGER_MODE_MANUAL)
Expand All @@ -40,36 +44,61 @@ if "allowed_contexts" in settings:
if "default_registry" in settings:
default_registry(settings.get("default_registry"))


# deploy CAPI
def deploy_capi():
version = settings.get("capi_version")
capi_uri = "https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version)
cmd = "curl -sSL {} | {} | {} apply -f -".format(capi_uri, envsubst_cmd, kubectl_cmd)
local(cmd, quiet = True)
capi_uri = "https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(
version
)
cmd = "curl -sSL {} | {} | {} apply -f -".format(
capi_uri, envsubst_cmd, kubectl_cmd
)
local(cmd, quiet=True)
if settings.get("extra_args"):
extra_args = settings.get("extra_args")
if extra_args.get("core"):
core_extra_args = extra_args.get("core")
if core_extra_args:
for namespace in ["capi-system"]:
patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args)
patch_args_with_extra_args(
namespace, "capi-controller-manager", core_extra_args
)
if extra_args.get("kubeadm-bootstrap"):
kb_extra_args = extra_args.get("kubeadm-bootstrap")
if kb_extra_args:
patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args)
patch_args_with_extra_args(
"capi-kubeadm-bootstrap-system",
"capi-kubeadm-bootstrap-controller-manager",
kb_extra_args,
)


def patch_args_with_extra_args(namespace, name, extra_args):
args_str = str(local("{} get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[0].args}}".format(kubectl_cmd, name, namespace)))
args_str = str(
local(
"{} get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[0].args}}".format(
kubectl_cmd, name, namespace
)
)
)
args_to_add = [arg for arg in extra_args if arg not in args_str]
if args_to_add:
args = args_str[1:-1].split()
args.extend(args_to_add)
patch = [{
"op": "replace",
"path": "/spec/template/spec/containers/0/args",
"value": args,
}]
local("{} patch deployment {} -n {} --type json -p='{}'".format(kubectl_cmd, name, namespace, str(encode_json(patch)).replace("\n", "")))
patch = [
{
"op": "replace",
"path": "/spec/template/spec/containers/0/args",
"value": args,
}
]
local(
"{} patch deployment {} -n {} --type json -p='{}'".format(
kubectl_cmd, name, namespace, str(encode_json(patch)).replace("\n", "")
)
)


# Users may define their own Tilt customizations in tilt.d. This directory is excluded from git and these files will
# not be checked in to version control.
Expand All @@ -78,23 +107,37 @@ def include_user_tilt_files():
for f in user_tiltfiles:
include(f)

def append_arg_for_container_in_deployment(yaml_stream, name, namespace, contains_image_name, args):

def append_arg_for_container_in_deployment(
yaml_stream, name, namespace, contains_image_name, args
):
for item in yaml_stream:
if item["kind"] == "Deployment" and item.get("metadata").get("name") == name and item.get("metadata").get("namespace") == namespace:
if (
item["kind"] == "Deployment"
and item.get("metadata").get("name") == name
and item.get("metadata").get("namespace") == namespace
):
containers = item.get("spec").get("template").get("spec").get("containers")
for container in containers:
if contains_image_name in container.get("image"):
container.get("args").extend(args)


def fixup_yaml_empty_arrays(yaml_str):
yaml_str = yaml_str.replace("conditions: null", "conditions: []")
return yaml_str.replace("storedVersions: null", "storedVersions: []")


def validate_auth():
substitutions = settings.get("kustomize_substitutions", {})
missing = [k for k in keys if k not in substitutions]
if missing:
fail("missing kustomize_substitutions keys {} in tilt-settings.json".format(missing))
fail(
"missing kustomize_substitutions keys {} in tilt-settings.json".format(
missing
)
)


tilt_helper_dockerfile_header = """
# Tilt image
Expand All @@ -113,35 +156,58 @@ COPY --from=tilt-helper /restart.sh .
COPY manager .
"""


# Build CAPG and add feature gates
def capg():
# Apply the kustomized yaml for this provider
substitutions = settings.get("kustomize_substitutions", {})
os.environ.update(substitutions)

# yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default
yaml = str(kustomizesub("./config/default"))
yaml = str(
kustomizesub("./hack/observability")
) # build an observable kind deployment by default
# TODO: consider to remove
# yaml = str(kustomizesub("./config/default"))

# add extra_args if they are defined
if settings.get("extra_args"):
gcp_extra_args = settings.get("extra_args").get("gcp")
if gcp_extra_args:
yaml_dict = decode_yaml_stream(yaml)
append_arg_for_container_in_deployment(yaml_dict, "capg-controller-manager", "capg-system", "cluster-api-gcp-controller", gcp_extra_args)
append_arg_for_container_in_deployment(
yaml_dict,
"capg-controller-manager",
"capg-system",
"cluster-api-gcp-controller",
gcp_extra_args,
)
yaml = str(encode_yaml_stream(yaml_dict))
yaml = fixup_yaml_empty_arrays(yaml)

# Set up a local_resource build of the provider's manager binary.
local_resource(
"manager",
cmd = 'mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager',
deps = ["api", "cloud", "config", "controllers", "exp", "feature", "pkg", "go.mod", "go.sum", "main.go"],
cmd="mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags '-extldflags \"-static\"' -o .tiltbuild/manager",
deps=[
"api",
"cloud",
"config",
"controllers",
"exp",
"feature",
"pkg",
"go.mod",
"go.sum",
"main.go",
],
)

dockerfile_contents = "\n".join([
tilt_helper_dockerfile_header,
tilt_dockerfile_header,
])
dockerfile_contents = "\n".join(
[
tilt_helper_dockerfile_header,
tilt_dockerfile_header,
]
)

entrypoint = ["sh", "/start.sh", "/manager"]
extra_args = settings.get("extra_args")
Expand All @@ -151,45 +217,110 @@ def capg():
# Set up an image build for the provider. The live update configuration syncs the output from the local_resource
# build into the container.
docker_build(
ref = "gcr.io/k8s-staging-cluster-api-gcp/cluster-api-gcp-controller",
context = "./.tiltbuild/",
dockerfile_contents = dockerfile_contents,
target = "tilt",
entrypoint = entrypoint,
only = "manager",
live_update = [
ref="gcr.io/k8s-staging-cluster-api-gcp/cluster-api-gcp-controller",
context="./.tiltbuild/",
dockerfile_contents=dockerfile_contents,
target="tilt",
entrypoint=entrypoint,
only="manager",
live_update=[
sync(".tiltbuild/manager", "/manager"),
run("sh /restart.sh"),
],
ignore = ["templates"],
ignore=["templates"],
)

k8s_yaml(blob(yaml))


def observability():
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

setting up OTEL Collector + Jaeger using helm chart

# Install the OpenTelemetry helm chart
gcp_project_id = os.getenv("GCP_PROJECT_ID", "")

k8s_yaml(
helm(
"./hack/observability/opentelemetry/chart",
name="opentelemetry-collector",
namespace="capg-system",
values=["./hack/observability/opentelemetry/values.yaml"],
# refer https://github.com/helm/helm/issues/1987
set=[
"extraEnvs[0].name=GCP_PROJECT_ID",
"extraEnvs[0].value=" + gcp_project_id,
],
)
)

k8s_yaml(
helm(
"./hack/observability/jaeger/chart",
name="jaeger-all-in-one",
namespace="capg-system",
set=[
# TODO: consider to remove
# "crd.install=false",
# "rbac.create=false",
"resources.limits.cpu=200m",
"resources.limits.memory=256Mi",
],
)
)

k8s_resource(
workload="jaeger-all-in-one",
new_name="traces: jaeger-all-in-one",
port_forwards=[
port_forward(16686, name="View traces", link_path="/search?service=capg")
],
labels=["observability"],
)

k8s_resource(workload="opentelemetry-collector", labels=["observability"])


def base64_encode(to_encode):
encode_blob = local("echo '{}' | tr -d '\n' | base64 - | tr -d '\n'".format(to_encode), quiet = True)
encode_blob = local(
"echo '{}' | tr -d '\n' | base64 - | tr -d '\n'".format(to_encode), quiet=True
)
return str(encode_blob)


def base64_encode_file(path_to_encode):
encode_blob = local("cat {} | tr -d '\n' | base64 - | tr -d '\n'".format(path_to_encode), quiet = True)
encode_blob = local(
"cat {} | tr -d '\n' | base64 - | tr -d '\n'".format(path_to_encode), quiet=True
)
return str(encode_blob)


def read_file_from_path(path_to_read):
str_blob = local("cat {} | tr -d '\n'".format(path_to_read), quiet = True)
str_blob = local("cat {} | tr -d '\n'".format(path_to_read), quiet=True)
return str(str_blob)


def base64_decode(to_decode):
decode_blob = local("echo '{}' | base64 --decode -".format(to_decode), quiet = True)
decode_blob = local("echo '{}' | base64 --decode -".format(to_decode), quiet=True)
return str(decode_blob)


def kustomizesub(folder):
yaml = local("hack/kustomize-sub.sh {}".format(folder), quiet = True)
yaml = local("hack/kustomize-sub.sh {}".format(folder), quiet=True)
return yaml


def waitforsystem():
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-bootstrap-system")
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system")
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")
local(
kubectl_cmd
+ " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-bootstrap-system"
)
local(
kubectl_cmd
+ " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system"
)
local(
kubectl_cmd
+ " wait --for=condition=ready --timeout=300s pod --all -n capi-system"
)


##############################
# Actual work happens here
Expand All @@ -208,4 +339,6 @@ deploy_capi()

capg()

observability()

waitforsystem()
Loading