Skip to content

Commit

Permalink
[stable-4.9] CI fixes (#2126)
Browse files Browse the repository at this point in the history
* cherry picked #2122 and in galaxy-importer
* Update rh-certified url to match 891656c
* Add sleeps and wait_for_all_tasks_gk calls, 2112, 2101
* Need to template out the nginx port also, 2118
No-Issue
  • Loading branch information
bmclaughlin authored May 1, 2024
1 parent 3f249ef commit 378f3de
Show file tree
Hide file tree
Showing 10 changed files with 203 additions and 91 deletions.
5 changes: 0 additions & 5 deletions galaxy_ng/tests/integration/api/test_artifact_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,11 +453,6 @@ def test_ansible_lint_exception_AAH_2606(ansible_config, upload_artifact, hub_ve
"meta/runtime.yml:1: yaml[new-line-at-end-of-file]:"
+ " No new line character at the end of file"
),
(
"tests/sanity/ignore-2.10.txt:1: sanity[cannot-ignore]:"
+ " Ignore file contains validate-modules:use-run-command-not-popen at line 1,"
+ " which is not a permitted ignore."
)
]

artifact = bc(
Expand Down
3 changes: 3 additions & 0 deletions galaxy_ng/tests/integration/api/test_namespace_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,8 @@ def test_namespace_edit_logo(galaxy_client):
}
my_namespace = gc.post("_ui/v1/my-namespaces/", body=payload)
assert my_namespace["avatar_url"] == ''
sleep(60)
wait_for_all_tasks_gk(gc)

namespaces = gc.get('_ui/v1/my-namespaces/')
name = my_namespace["name"]
Expand All @@ -198,6 +200,7 @@ def test_namespace_edit_logo(galaxy_client):
"avatar_url": "https://avatars.githubusercontent.com/u/1869705?v=4"
}
gc.put(f"_ui/v1/my-namespaces/{name}/", body=payload)
sleep(60)
wait_for_all_tasks_gk(gc)
updated_namespace = gc.get(f'_ui/v1/my-namespaces/{name}/')
assert updated_namespace["avatar_url"] != ""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ def setUp(self):
super().setUp()
self.remote_data = {
"name": "rh-certified",
"url": "https://console.redhat.com/api/automation-hub/",
"url": "https://console.redhat.com/api/automation-hub/content/published/",
}
self.remote = CollectionRemote.objects.get(name=self.remote_data["name"])
self.repository = AnsibleRepository.objects.get(name=self.remote_data["name"])
Expand Down
4 changes: 4 additions & 0 deletions profiles/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ WORKDIR /opt/galaxy_ng/

RUN switch_python 3.11

# override the image's nginx templating scripts ...
RUN cp -p /opt/galaxy_ng/profiles/base/nginx/template_nginx.py /nginx/.
RUN cp /opt/galaxy_ng/profiles/base/nginx/nginx.conf.j2 /nginx/.

# preinstall galaxy_ng in thebase image
RUN python3.11 -m pip install .

Expand Down
146 changes: 146 additions & 0 deletions profiles/base/nginx/nginx.conf.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
# TODO: Support IPv6.
# TODO: Maybe serve multiple `location`s, not just one.

# The "nginx" package on fedora creates this user and group.
user nginx nginx;
# Gunicorn docs suggest this value.
worker_processes 1;
daemon off;
events {
worker_connections 1024; # increase if you have lots of clients
accept_mutex off; # set to 'on' if nginx worker_processes > 1
}

http {
include mime.types;
# fallback in case we can't determine a type
default_type application/octet-stream;
sendfile on;

# If left at the default of 1024, nginx emits a warning about being unable
# to build optimal hash types.
types_hash_max_size 4096;

upstream pulp-content {
server 127.0.0.1:24816;
}

upstream pulp-api {
server 127.0.0.1:24817;
}

server {
# Gunicorn docs suggest the use of the "deferred" directive on Linux.
{% if https | default(false) -%}
listen 443 default_server deferred ssl;

ssl_certificate /etc/pulp/certs/pulp_webserver.crt;
ssl_certificate_key /etc/pulp/certs/pulp_webserver.key;
ssl_session_cache shared:SSL:50m;
ssl_session_timeout 1d;
ssl_session_tickets off;

# intermediate configuration
ssl_protocols TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;

# HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
add_header Strict-Transport-Security max-age=15768000;
{%- else -%}
listen {{ NGINX_PORT }} default_server deferred;
{%- endif %}
server_name $hostname;

# The default client_max_body_size is 1m. Clients uploading
# files larger than this will need to chunk said files.
client_max_body_size 10m;

# Gunicorn docs suggest this value.
keepalive_timeout 5;

#location {{ content_path }} {
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# proxy_set_header X-Forwarded-Proto $scheme;
# proxy_set_header Host $http_host;
# # we don't want nginx trying to do something clever with
# # redirects, we set the Host: header above already.
# proxy_redirect off;
# proxy_pass http://pulp-content;
#}

location /pulp/content/ {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://pulp-content;
}

location {{ api_root }}api/v3/ {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://pulp-api;
client_max_body_size 0;
}

{%- if domain_enabled | default(false) %}
location ~ {{ api_root }}.+/api/v3/ {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://pulp-api;
client_max_body_size 0;
}
{%- endif %}

location /auth/login/ {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://pulp-api;
}

include pulp/*.conf;

location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://pulp-api;
# most pulp static files are served through whitenoise
# http://whitenoise.evans.io/en/stable/
}

{%- if https | default(false) %}
# ACME http-01 tokens, i.e, for Let's Encrypt
location /.well-known/ {
try_files $uri $uri/ =404;
}
{%- endif %}
}
{%- if https | default(false) %}
server {
listen 55001 default_server;
server_name _;
return 301 https://$host$request_uri;
}
{%- endif %}
}


39 changes: 39 additions & 0 deletions profiles/base/nginx/template_nginx.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import argparse
import os
import django
from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured

from jinja2 import Template


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Create Pulp's nginx conf file based on current settings.",
)
parser.add_argument("template_file", type=open)
parser.add_argument("output_file", type=argparse.FileType("w"))
args = parser.parse_args()

https = os.getenv("PULP_HTTPS", "false")
values = {
"https": https.lower() == "true",
"api_root": "/pulp/",
"content_path": "/pulp/content/",
"domain_enabled": False,
}

try:
django.setup()
from django.conf import settings
except (AppRegistryNotReady, ImproperlyConfigured):
print("Failed to find settings for nginx template, using defaults")
else:
values["api_root"] = settings.API_ROOT
values["content_path"] = settings.CONTENT_PATH_PREFIX
values["domain_enabled"] = getattr(settings, "DOMAIN_ENABLED", False)

values['NGINX_PORT'] = os.environ.get('NGINX_PORT', '55001')

template = Template(args.template_file.read())
output = template.render(**values)
args.output_file.write(output)
31 changes: 3 additions & 28 deletions requirements/requirements.common.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#
# This file is autogenerated by pip-compile with python 3.8
# To update, run:
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile --output-file=requirements/requirements.common.txt setup.py
#
Expand All @@ -24,11 +24,6 @@ asgiref==3.7.2
# via django
async-lru==2.0.4
# via pulp-ansible
async-timeout==4.0.3
# via
# aiohttp
# pulpcore
# redis
asyncio-throttle==1.0.2
# via pulpcore
attrs==22.2.0
Expand All @@ -42,11 +37,6 @@ backoff==2.2.1
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
# pulpcore
backports-zoneinfo==0.2.1 ; python_version < "3.9"
# via
# django
# galaxy-ng (setup.py)
# psycopg
bindep==2.11.0
# via ansible-builder
black==24.1.0
Expand Down Expand Up @@ -183,13 +173,8 @@ idna==3.6
# yarl
importlib-metadata==6.0.1
# via
# markdown
# opentelemetry-api
# pulpcore
importlib-resources==6.1.1
# via
# jsonschema
# pulp-glue
inflection==0.5.1
# via drf-spectacular
insights-analytics-collector==0.3.2
Expand Down Expand Up @@ -311,8 +296,6 @@ pbr==6.0.0
# via bindep
pillow==10.0.1
# via pulp-ansible
pkgutil-resolve-name==1.3.10
# via jsonschema
platformdirs==4.1.0
# via black
prometheus-client==0.19.0
Expand Down Expand Up @@ -438,18 +421,12 @@ subprocess-tee==0.4.1
# via ansible-lint
tablib[html,ods,xls,xlsx,yaml]==3.5.0
# via django-import-export
tomli==2.0.1
# via black
types-setuptools==69.0.0.20240125
# via requirements-parser
typing-extensions==4.9.0
# via
# asgiref
# async-lru
# black
# opentelemetry-sdk
# psycopg
# rich
uritemplate==4.1.1
# via drf-spectacular
url-normalize==1.4.3
Expand Down Expand Up @@ -481,9 +458,7 @@ yarl==1.9.4
# aiohttp
# pulpcore
zipp==3.17.0
# via
# importlib-metadata
# importlib-resources
# via importlib-metadata

# The following packages are considered to be unsafe in a requirements file:
# setuptools
Loading

0 comments on commit 378f3de

Please sign in to comment.