Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix get_tempauth_token host keyerror #16

Merged
merged 1 commit into from
Jul 19, 2023
Merged

fix get_tempauth_token host keyerror #16

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 30 additions & 38 deletions scripts/generate_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
#!/usr/bin/env python3

import argparse
import datetime
import hashlib
import io
import json
import os
import pathlib
import random
import socket
import time
import typing
from urllib.parse import quote

import lorem
Expand All @@ -22,13 +17,13 @@ def build_path(*parts: str) -> str:
return "/" + "/".join(quote(part, safe="") for part in parts)


def wait_for_port(url, timeout=5.0, verbose=False, quiet=False):
def wait_for_port(url, timeout=5.0, verbose=False):
start_time = time.perf_counter()

while True:
try:
r = requests.head(url)
if verbose and not quiet:
if verbose:
print()
print(f"{url} seems to be accepting connections")
break
Expand All @@ -42,16 +37,16 @@ def wait_for_port(url, timeout=5.0, verbose=False, quiet=False):
) from ex
time.sleep(1)
end = time.perf_counter() - start_time
if end > 1 and not quiet:
if end > 1 and verbose:
print(f"Waited {end} for {url}", end="\r")
print()


def get_tempauth_token(
host="localhost", port=8080, username="test:tester", password="testing"
) -> str:
) -> typing.Tuple[str, str]:
auth = requests.get(
swift_url_template(host, port),
swift_url_template(host=host, port=port),
headers={
"X-Storage-User": username,
"X-Storage-Pass": password,
Expand All @@ -68,7 +63,7 @@ def get_keystone_token(
username="swift",
password="veryfast",
project="service",
) -> str:
) -> typing.Tuple[str, str]:
keystone_url = keystone_url_template(host=host, port=port)

auth_data = {
Expand Down Expand Up @@ -119,7 +114,7 @@ def get_all_containers_obj_count(swift_url: str, token: str) -> int:
return total


def create_from_lorem(n_containers, n_objects):
def create_from_lorem(n_containers, n_objects) -> list:
n_container_tags = 3
n_object_tags = 4
data = []
Expand Down Expand Up @@ -164,9 +159,10 @@ def create_from_lorem(n_containers, n_objects):


def populate_swift(
swift_url: str, token: str, data: dict, subfolder="", verbose=False, quiet=False
swift_url: str, token: str, data: list, subfolder="", verbose=False
):
if not quiet:
start: float = 0
if verbose:
start = time.perf_counter()
print()
n_containers = len(data)
Expand All @@ -182,7 +178,7 @@ def populate_swift(
r = requests.put(f"{swift_url}/{container_name}", headers=headers)
if r.status_code not in {201, 202}:
print(f"ERROR {r.status_code} {container_name}")
if verbose and not quiet:
if verbose:
print(f"{r.status_code} {container_name}")
for obj in container["objects"]:
obj_name = subfolder + obj["name"]
Expand All @@ -198,9 +194,9 @@ def populate_swift(
)
if r.status_code != 201:
print(f"ERROR {r.status_code} {obj_name}")
if verbose and not quiet:
if verbose:
print(f"{r.status_code} {obj_name}")
if not quiet:
if verbose:
end = time.perf_counter() - start
print(f"Done in {end:.2f} seconds.")

Expand All @@ -215,26 +211,26 @@ def run(
timeout=60,
runs=2,
verbose=False,
quiet=False,
):
data: list
for _ in range(0, runs):
start_timeout = time.perf_counter()
existing_objs = get_account_obj_count(swift_url, token)
containers_obj_count = 0
if json_path:
with open(json_path, "r") as fp:
data = json.load(fp)
if not quiet:
if verbose:
print(f"Populating data from {json_path}")
n_containers = len(data)
n_objects = len(data[0]["objects"])
else:
data = create_from_lorem(n_containers, n_objects)

containers_obj_count = get_all_containers_obj_count(swift_url, token)
populate_swift(swift_url, token, data, subfolder, verbose, quiet)
populate_swift(swift_url, token, data, subfolder, verbose)

if verbose and not quiet:
if verbose:
print()
print("Wait until metadata updates")

Expand All @@ -249,7 +245,7 @@ def run(
if obj == containers_obj_count:
break
current = time.perf_counter() - start
if not quiet:
if verbose:
print(
f"Current: {obj}, expected: {expected}, started with {existing_objs}. Created {containers_obj_count - existing_objs} new objects. Waited for {current:.2f} seconds",
end="\r",
Expand All @@ -258,7 +254,7 @@ def run(
raise TimeoutError("Waited too long for metadata to update")
time.sleep(1)
end = time.perf_counter() - start
if not quiet:
if verbose:
print(
f"We got {obj} objects, expected: {expected}, started with {existing_objs}. Created {containers_obj_count - existing_objs} new objects. Done in {end:.2f} seconds", end="\r"
)
Expand Down Expand Up @@ -340,29 +336,27 @@ def run(

outtput_group = parser.add_mutually_exclusive_group()
outtput_group.add_argument(
"-v", "--verbose", action="store_true", help="increase output verbosity"
"-v", "--verbose", dest="verbosity", action="store_true", help="increase output verbosity"
)
outtput_group.add_argument(
"-q", "--quiet", action="store_true", help="Don't print to console"
"-q", "--quiet", dest="verbosity", action="store_false", help="Don't print to console"
)

args = parser.parse_args()

if not args.quiet:
total_start: float = 0
if args.verbosity:
total_start = time.perf_counter()

if args.keystone:
wait_for_port(
keystone_url_template(host=args.host, port=args.keystone_port),
timeout=args.timeout,
verbose=args.verbose,
quiet=args.quiet,
verbose=args.verbosity,
)
wait_for_port(
swift_url_template(host=args.host, port=args.swift_port),
timeout=args.timeout,
verbose=args.verbose,
quiet=args.quiet,
verbose=args.verbosity,
)
swift_url, token = get_keystone_token(
args.host, args.keystone_port, args.username, args.password, args.project
Expand All @@ -371,14 +365,13 @@ def run(
wait_for_port(
swift_url_template(host=args.host, port=args.swift_port),
timeout=args.timeout,
verbose=args.verbose,
quiet=args.quiet,
verbose=args.verbosity,
)
swift_url, token = get_tempauth_token(
args.host, args.swift_port, args.username, args.password
)

if args.verbose and not args.quiet:
if args.verbosity:
print(f"Got token: {token}. Swift url: {swift_url}")
print(f"Generating data ...")

Expand All @@ -391,11 +384,10 @@ def run(
args.objects,
args.timeout,
args.runs,
verbose=args.verbose,
quiet=args.quiet,
verbose=args.verbosity,
)

if not args.quiet:
if args.verbosity:
total_end = time.perf_counter() - total_start
print()
print(f"Completed in {total_end:.2f} seconds")