diff --git a/extensions/commands/art/README.md b/extensions/commands/art/README.md index 3affd26..ce90085 100644 --- a/extensions/commands/art/README.md +++ b/extensions/commands/art/README.md @@ -96,7 +96,7 @@ Then upload the created package to your repository: conan upload ... -c -r ... ``` -Using the generated JSON files you can create a BuildInfo JSON. To do this, you need to provide the build +Using the generated JSON files you can create a Build Info JSON file. To do this, you need to provide the build name and number. You will also need to indicate the artifactory server to use: ``` diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 38b005b..5a71535 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -1,4 +1,3 @@ -import base64 import datetime import json import os @@ -6,8 +5,6 @@ import hashlib from pathlib import Path -import requests - from conan.api.conan_api import ConanAPI from conan.api.output import cli_out_write from conan.cli.command import conan_command, conan_subcommand @@ -16,58 +13,12 @@ from conan import conan_version from conan.tools.scm import Version -from cmd_server import get_server - - -def response_to_str(response): - content = response.content - try: - # A bytes message, decode it as str - if isinstance(content, bytes): - content = content.decode('utf-8') - - content_type = response.headers.get("content-type") - - if content_type == "application/json": - # Errors from Artifactory looks like: - # {"errors" : [ {"status" : 400, "message" : "Bla bla bla"}]} - try: - data = json.loads(content)["errors"][0] - content = "{}: {}".format(data["status"], data["message"]) - except Exception: - pass - elif "text/html" in content_type: - content = "{}: {}".format(response.status_code, response.reason) - - return content - except Exception: - return response.content - - -def api_request(method, request_url, user=None, password=None, json_data=None, - sign_key_name=None): - headers = {} - if json_data: - headers.update({"Content-Type": "application/json"}) - if sign_key_name: - headers.update({"X-JFrog-Crypto-Key-Name": sign_key_name}) - - requests_method = getattr(requests, method) - if user and password: - response = requests_method(request_url, auth=( - user, password), data=json_data, headers=headers) - else: - response = requests_method(request_url) - - if response.status_code == 401: - raise Exception(response_to_str(response)) - elif response.status_code not in [200, 204]: - raise Exception(response_to_str(response)) - - return response_to_str(response) +from utils import api_request, assert_server_or_url_user_password +from cmd_property import get_properties, set_properties +from cmd_server import get_url_user_password -def get_remote_path(rrev, package_id=None, prev=None): +def _get_remote_path(rrev, package_id=None, prev=None): ref = RecipeReference.loads(rrev) user = ref.user or "_" channel = ref.channel or "_" @@ -79,7 +30,7 @@ def get_remote_path(rrev, package_id=None, prev=None): return f"{rev_path}/package/{package_id}/{prev}" -def get_hashes(file_path): +def _get_hashes(file_path): buf_size = 65536 md5 = hashlib.md5() @@ -97,7 +48,7 @@ def get_hashes(file_path): return md5.hexdigest(), sha1.hexdigest(), sha256.hexdigest() -def get_formatted_time(): +def _get_formatted_time(): now = datetime.datetime.now(datetime.timezone.utc) local_tz_offset = now.astimezone().strftime('%z') formatted_time = now.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + local_tz_offset @@ -113,7 +64,7 @@ def get_formatted_time(): return formatted_time -def get_requested_by(nodes, node_id, artifact_type): +def _get_requested_by(nodes, node_id, artifact_type): node_id = str(node_id) root_direct = [] @@ -148,7 +99,7 @@ def get_requested_by(nodes, node_id, artifact_type): return ret -class BuildInfo: +class _BuildInfo: def __init__(self, graph, name, number, repository, with_dependencies=False, url=None, user=None, password=None): @@ -174,10 +125,10 @@ def get_artifacts(self, node, artifact_type, is_dependency=False): if artifact_type == "recipe": artifacts_names = ["conan_sources.tgz", "conan_export.tgz", "conanfile.py", "conanmanifest.txt"] - remote_path = get_remote_path(node.get('ref')) + remote_path = _get_remote_path(node.get('ref')) else: artifacts_names = ["conan_package.tgz", "conaninfo.txt", "conanmanifest.txt"] - remote_path = get_remote_path(node.get('ref'), node.get("package_id"), node.get("prev")) + remote_path = _get_remote_path(node.get('ref'), node.get("package_id"), node.get("prev")) def _get_local_artifacts(): local_artifacts = [] @@ -188,7 +139,7 @@ def _get_local_artifacts(): for file_path in dl_folder.glob("*"): if file_path.is_file(): file_name = file_path.name - md5, sha1, sha256 = get_hashes(file_path) + md5, sha1, sha256 = _get_hashes(file_path) artifact_info = {"type": os.path.splitext(file_name)[1].lstrip('.'), "sha256": sha256, "sha1": sha1, @@ -261,7 +212,7 @@ def _get_remote_artifacts(): # complete the information for the artifacts: if is_dependency: - requested_by = get_requested_by(self._graph["graph"]["nodes"], node.get("id"), artifact_type) + requested_by = _get_requested_by(self._graph["graph"]["nodes"], node.get("id"), artifact_type) for artifact in artifacts: artifact.update({"requestedBy": requested_by}) @@ -326,7 +277,7 @@ def header(self): "name": self._name, "number": self._number, "agent": {}, - "started": get_formatted_time(), + "started": _get_formatted_time(), "buildAgent": {"name": "conan", "version": f"{str(conan_version)}"}} def create(self): @@ -335,7 +286,7 @@ def create(self): return json.dumps(bi, indent=4) -def manifest_from_build_info(build_info, repository, with_dependencies=True): +def _manifest_from_build_info(build_info, repository, with_dependencies=True): manifest = {"files": []} for module in build_info.get("modules"): for artifact in module.get("artifacts"): @@ -350,35 +301,16 @@ def manifest_from_build_info(build_info, repository, with_dependencies=True): if ":" in full_reference: pkgid = full_reference.split(":")[1].split("#")[0] prev = full_reference.split(":")[1].split("#")[1] - full_path = repository + "/" + get_remote_path(rrev, pkgid, prev) + "/" + filename + full_path = repository + "/" + _get_remote_path(rrev, pkgid, prev) + "/" + filename if not any(d['path'] == full_path for d in manifest["files"]): manifest["files"].append({"path": full_path, "checksum": dependency.get("sha256")}) return manifest -def assert_server_or_url_user_password(args): - if args.server and args.url: - raise ConanException("--server and --url (with --user & --password) flags cannot be used together.") - if not args.server and not args.url: - raise ConanException("Specify --server or --url (with --user & --password) flags to contact Artifactory.") - if args.url: - if not args.user or not args.password: - raise ConanException("Specify --user and --password to use with the --url flag to contact Artifactory.") - assert args.server or (args.url and args.user and args.password) - - -def get_url_user_password(args): - if args.server: - server_name = args.server.strip() - server = get_server(server_name) - url = server.get("url") - user = server.get("user") - password = server.get("password") - else: - url = args.url - user = args.user - password = args.password - return url, user, password +def _check_min_required_conan_version(min_ver): + if conan_version < Version(min_ver): + raise ConanException("This custom command is only compatible with " \ + f"Conan versions>={min_ver}. Please update Conan.") def _add_default_arguments(subparser, is_bi_create=False): @@ -393,19 +325,13 @@ def _add_default_arguments(subparser, is_bi_create=False): return subparser -@conan_command(group="Custom commands") +@conan_command(group="Artifactory commands") def build_info(conan_api: ConanAPI, parser, *args): """ - Manages JFROG BuildInfo + Manages JFrog Build Info (https://www.buildinfo.org/) """ -def check_min_required_conan_version(min_ver): - if conan_version < Version(min_ver): - raise ConanException("This custom command is only compatible with " \ - f"Conan versions>={min_ver}. Please update Conan.") - - @conan_subcommand() def build_info_create(conan_api: ConanAPI, parser, subparser, *args): """ @@ -413,7 +339,7 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): """ _add_default_arguments(subparser, is_bi_create=True) - check_min_required_conan_version("2.0.6") + _check_min_required_conan_version("2.0.6") subparser.add_argument("json", help="Conan generated JSON output file.") subparser.add_argument("build_name", help="Build name property for BuildInfo.") @@ -432,8 +358,8 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): # remove the 'conanfile' node data["graph"]["nodes"].pop("0") - bi = BuildInfo(data, args.build_name, args.build_number, args.repository, - with_dependencies=args.with_dependencies, url=url, user=user, password=password) + bi = _BuildInfo(data, args.build_name, args.build_number, args.repository, + with_dependencies=args.with_dependencies, url=url, user=user, password=password) cli_out_write(bi.create()) @@ -455,10 +381,7 @@ def build_info_upload(conan_api: ConanAPI, parser, subparser, *args): with open(args.build_info) as f: build_info_json = json.load(f) - # FIXME: this code is repeated in the art:property command, - # we have to fix that custom commands can share modules between them - - # first, set the properties build.name and build.number + # first, set the properties build.name and build.number # for the artifacts in the BuildInfo build_name = build_info_json.get("name") @@ -466,20 +389,13 @@ def build_info_upload(conan_api: ConanAPI, parser, subparser, *args): for module in build_info_json.get('modules'): for artifact in module.get('artifacts'): - artifact_properties = {} artifact_path = artifact.get('path') - try: - request_url = f"{url}/api/storage/{artifact_path}?properties" - props_response = api_request("get", request_url, user, password) - artifact_properties = json.loads(props_response).get("properties") - except: - pass + artifact_properties = get_properties(artifact_path, url, user, password) artifact_properties.setdefault("build.name", []).append(build_name) artifact_properties.setdefault("build.number", []).append(build_number) - request_url = f"{url}/api/metadata/{artifact_path}" - api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) + set_properties(artifact_properties, artifact_path, url, user, password, False) # now upload the BuildInfo request_url = f"{url}/api/build" @@ -622,7 +538,7 @@ def build_info_append(conan_api: ConanAPI, parser, subparser, *args): if not any(d['id'] == module.get('id') for d in all_modules): all_modules.append(module) - bi = BuildInfo(None, args.build_name, args.build_number, None) + bi = _BuildInfo(None, args.build_name, args.build_number, None) bi_json = bi.header() bi_json.update({"modules": all_modules}) cli_out_write(json.dumps(bi_json, indent=4)) @@ -652,7 +568,7 @@ def build_info_create_bundle(conan_api: ConanAPI, parser, subparser, *args): with open(args.json, 'r') as f: data = json.load(f) - manifest = manifest_from_build_info(data, args.repository, with_dependencies=True) + manifest = _manifest_from_build_info(data, args.repository, with_dependencies=True) bundle_json = { "payload": manifest diff --git a/extensions/commands/art/cmd_property.py b/extensions/commands/art/cmd_property.py index 9207fb0..b046be6 100644 --- a/extensions/commands/art/cmd_property.py +++ b/extensions/commands/art/cmd_property.py @@ -1,8 +1,4 @@ -import base64 import json -import os - -import requests from conan.api.conan_api import ConanAPI from conan.cli.command import conan_command, conan_subcommand @@ -10,7 +6,27 @@ from conans.model.package_ref import PkgReference from conan.errors import ConanException -from cmd_build_info import api_request, assert_server_or_url_user_password, get_url_user_password +from utils import api_request, assert_server_or_url_user_password +from cmd_server import get_url_user_password + + +def set_properties(properties, path, url, user, password, recursive): + json_data = json.dumps({"props": properties}) + recursive = "1" if recursive else "0" + request_url = f"{url}/api/metadata/{path}?&recursiveProperties={recursive}" + api_request("patch", request_url, user, password, json_data=json_data) + + +def get_properties(path, url, user, password): + properties = {} + request_url = f"{url}/api/storage/{path}?properties" + + try: + props_response = api_request("get", request_url, user, password) + properties = json.loads(props_response).get("properties") + except: + pass + return properties def _get_path_from_ref(ref): @@ -40,10 +56,10 @@ def _add_default_arguments(subparser): return subparser -@conan_command(group="Custom commands") +@conan_command(group="Artifactory commands") def property(conan_api: ConanAPI, parser, *args): """ - Manages artifacts properties in Artifactory. + Manages Conan packages properties in Artifactory. """ @@ -76,36 +92,29 @@ def property_add(conan_api: ConanAPI, parser, subparser, *args): data = json.loads(api_request("get", request_url, user, password)) + # just consider those artifacts that have conan in the name # conan_artifacts = [artifact for artifact in data.get("files") if "conan" in artifact.get('uri')] # get properties for all artifacts for artifact in data.get("files"): uri = artifact.get('uri') + path = f"{args.repository}/{root_path}{uri}" - request_url = f"{url}/api/storage/{args.repository}/{root_path}{uri}?properties" - - artifact_properties = {} - - try: - props_response = api_request("get", request_url, user, password) - artifact_properties = json.loads(props_response).get("properties") - except: - pass + artifact_properties = get_properties(path, url, user, password) for property in args.property: key, val = property.split('=')[0], property.split('=')[1] artifact_properties.setdefault(key, []).append(val) if artifact_properties: - request_url = f"{url}/api/metadata/{args.repository}/{root_path}{uri}?&recursiveProperties=0" - api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) + set_properties(artifact_properties, path, url, user, password, False) @conan_subcommand() def property_set(conan_api: ConanAPI, parser, subparser, *args): """ - Set properties for artifacts under a Conan reference recursively. + Set properties for artifacts under a Conan reference. """ _add_default_arguments(subparser) @@ -118,64 +127,11 @@ def property_set(conan_api: ConanAPI, parser, subparser, *args): if not args.property: raise ConanException("Please, add at least one property with the --property argument.") - recursive = "1" if args.recursive else "0" - - json_data = json.dumps( - {"props": {prop.split('=')[0]: prop.split('=')[1] for prop in args.property}}) - - url, user, password = get_url_user_password(args) - request_url = f"{url}/api/metadata/{args.repository}/{_get_path_from_ref(args.reference)}?&recursiveProperties={recursive}" - - api_request("patch", request_url, user, password, json_data=json_data) - - -@conan_subcommand() -def property_build_info_add(conan_api: ConanAPI, parser, subparser, *args): - """ - Load a Build Info JSON and add the build.number and build.name properties to all the artifacts present in the JSON. - You can also add arbitrary properties with the --property argument. - """ - - subparser.add_argument("json", help="Build Info JSON.") - - subparser.add_argument("--property", action='append', - help='Property to add, like --property="key1=value1" --property="key2=value2". \ - If the property already exists, the values are appended.') - - subparser.add_argument("--server", help="Server name of the Artifactory to get the build info from") - subparser.add_argument("--url", help="Artifactory url, like: https://
/artifactory") - subparser.add_argument("--user", help="user name for the repository") - subparser.add_argument("--password", help="password for the user name") - - args = parser.parse_args(*args) - assert_server_or_url_user_password(args) - - with open(args.json, 'r') as f: - data = json.load(f) - - build_name = data.get("name") - build_number = data.get("number") - + # json_data = json.dumps( + # {"props": {prop.split('=')[0]: prop.split('=')[1] for prop in args.property}}) + properties = {prop.split('=')[0]: prop.split('=')[1] for prop in args.property} + path = f"{args.repository}/{_get_path_from_ref(args.reference)}" url, user, password = get_url_user_password(args) + recursive = "1" if args.recursive else "0" - for module in data.get('modules'): - for artifact in module.get('artifacts'): - artifact_properties = {} - artifact_path = artifact.get('path') - try: - request_url = f"{url}/api/storage/{artifact_path}?properties" - props_response = api_request("get", request_url, user, password) - artifact_properties = json.loads(props_response).get("properties") - except: - pass - - artifact_properties.setdefault("build.name", []).append(build_name) - artifact_properties.setdefault("build.number", []).append(build_number) - - if args.property: - for property in args.property: - key, val = property.split('=')[0], property.split('=')[1] - artifact_properties.setdefault(key, []).append(val) - - request_url = f"{url}/api/metadata/{artifact_path}" - api_request("patch", request_url, user, password, json_data=json.dumps({"props": artifact_properties})) + set_properties(properties, path, url, user, password, recursive) diff --git a/extensions/commands/art/cmd_server.py b/extensions/commands/art/cmd_server.py index 4adae54..17aee0f 100644 --- a/extensions/commands/art/cmd_server.py +++ b/extensions/commands/art/cmd_server.py @@ -2,62 +2,32 @@ import getpass import json import os.path -import requests from conan.api.conan_api import ConanAPI from conan.api.output import ConanOutput, cli_out_write from conan.cli.command import conan_command, conan_subcommand from conan.errors import ConanException +from utils import api_request + SERVERS_FILENAME = ".art-servers" -def response_to_str(response): - content = response.content - try: - # A bytes message, decode it as str - if isinstance(content, bytes): - content = content.decode() - - content_type = response.headers.get("content-type") - - if content_type == "application/json": - # Errors from Artifactory looks like: - # {"errors" : [ {"status" : 400, "message" : "Bla bla bla"}]} - try: - data = json.loads(content)["errors"][0] - content = "{}: {}".format(data["status"], data["message"]) - except Exception: - pass - elif "text/html" in content_type: - content = "{}: {}".format(response.status_code, response.reason) - - return content - except Exception: - return response.content - - -def api_request(type, request_url, user=None, password=None, json_data=None): - headers = {} - if json_data: - headers.update({"Content-Type": "application/json"}) - - requests_method = getattr(requests, type) - if user and password: - response = requests_method(request_url, auth=( - user, password), data=json_data, headers=headers) +def get_url_user_password(args): + if args.server: + server_name = args.server.strip() + server = _get_server(server_name) + url = server.get("url") + user = server.get("user") + password = server.get("password") else: - response = requests_method(request_url) - - if response.status_code == 401: - raise ConanException(response_to_str(response)) - elif response.status_code not in [200, 204]: - raise ConanException(response_to_str(response)) - - return response_to_str(response) + url = args.url + user = args.user + password = args.password + return url, user, password -def get_server(server_name): +def _get_server(server_name): servers = _read_servers() server_names = [s["name"] for s in servers] if server_name not in server_names: @@ -105,7 +75,7 @@ def _add_default_arguments(subparser): return subparser -@conan_command(group="Custom commands") +@conan_command(group="Artifactory commands") def server(conan_api: ConanAPI, parser, *args): """ Manages Artifactory server and credentials. diff --git a/extensions/commands/art/utils.py b/extensions/commands/art/utils.py new file mode 100644 index 0000000..457651c --- /dev/null +++ b/extensions/commands/art/utils.py @@ -0,0 +1,63 @@ +import json +import requests + +from conan.errors import ConanException + + +def response_to_str(response): + content = response.content + try: + # A bytes message, decode it as str + if isinstance(content, bytes): + content = content.decode('utf-8') + + content_type = response.headers.get("content-type") + + if content_type == "application/json": + # Errors from Artifactory looks like: + # {"errors" : [ {"status" : 400, "message" : "Bla bla bla"}]} + try: + data = json.loads(content)["errors"][0] + content = "{}: {}".format(data["status"], data["message"]) + except Exception: + pass + elif "text/html" in content_type: + content = "{}: {}".format(response.status_code, response.reason) + + return content + except Exception: + return response.content + + +def api_request(method, request_url, user=None, password=None, json_data=None, + sign_key_name=None): + headers = {} + if json_data: + headers.update({"Content-Type": "application/json"}) + if sign_key_name: + headers.update({"X-JFrog-Crypto-Key-Name": sign_key_name}) + + requests_method = getattr(requests, method) + if user and password: + response = requests_method(request_url, auth=( + user, password), data=json_data, headers=headers) + else: + response = requests_method(request_url) + + if response.status_code == 401: + raise Exception(response_to_str(response)) + elif response.status_code not in [200, 204]: + raise Exception(response_to_str(response)) + + return response_to_str(response) + + +def assert_server_or_url_user_password(args): + if args.server and args.url: + raise ConanException("--server and --url (with --user & --password) flags cannot be used together.") + if not args.server and not args.url: + raise ConanException("Specify --server or --url (with --user & --password) flags to contact Artifactory.") + if args.url: + if not args.user or not args.password: + raise ConanException("Specify --user and --password to use with the --url flag to contact Artifactory.") + assert args.server or (args.url and args.user and args.password)