From a09b8b815db0e1460b6ef81c40dc23bfeda76513 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Sat, 3 Aug 2024 23:10:34 +0000 Subject: [PATCH 1/4] Use jinja for html --- ydb/ci/build_bloat/html/index.html | 6 +- ydb/ci/build_bloat/html/webtreemap.css | 12 +- .../build_bloat/html_template_bloat/bloat.css | 57 ---- .../html_template_bloat/index.html | 33 --- .../html_template_bloat/webtreemap.css | 65 ----- .../html_template_bloat/webtreemap.js | 261 ------------------ ydb/ci/build_bloat/main.py | 19 +- ydb/ci/build_bloat/template_bloat.py | 22 +- 8 files changed, 41 insertions(+), 434 deletions(-) delete mode 100644 ydb/ci/build_bloat/html_template_bloat/bloat.css delete mode 100644 ydb/ci/build_bloat/html_template_bloat/index.html delete mode 100644 ydb/ci/build_bloat/html_template_bloat/webtreemap.css delete mode 100644 ydb/ci/build_bloat/html_template_bloat/webtreemap.js diff --git a/ydb/ci/build_bloat/html/index.html b/ydb/ci/build_bloat/html/index.html index ce493d74b95a..1c352ebaf5e4 100644 --- a/ydb/ci/build_bloat/html/index.html +++ b/ydb/ci/build_bloat/html/index.html @@ -12,9 +12,9 @@
Click on a box to zoom in. Click on the outermost box to zoom out.
Legend:
-
Dir
-
Cpp
-
Header
+ {% for id_, name, color in types %} +
{{ name }}
+ {% endfor %}
diff --git a/ydb/ci/build_bloat/html/webtreemap.css b/ydb/ci/build_bloat/html/webtreemap.css index d8f1a3c950a3..81110b5e1f06 100644 --- a/ydb/ci/build_bloat/html/webtreemap.css +++ b/ydb/ci/build_bloat/html/webtreemap.css @@ -18,15 +18,11 @@ } /* Optional: Different background colors depending on type. */ -.webtreemap-type-h { - background: #66C2A5; -} -.webtreemap-type-cpp { - background: #FC8D62; -} -.webtreemap-type-dir { - background: #8DA0CB; +{% for id_, name, color in types %} +.webtreemap-type-{{ id_ }} { + background: {{ color }}; } +{% endfor %} #legend > * { border: solid 1px #444; diff --git a/ydb/ci/build_bloat/html_template_bloat/bloat.css b/ydb/ci/build_bloat/html_template_bloat/bloat.css deleted file mode 100644 index 3fcca972c70c..000000000000 --- a/ydb/ci/build_bloat/html_template_bloat/bloat.css +++ /dev/null @@ -1,57 +0,0 @@ -/* Based on https://github.com/martine/webtreemap */ - -body { - font-family: sans-serif; - font-size: 0.8em; - height: calc(100vh - 20px); -} - -.content { - height: 100%; - display: flex; - flex-direction: column; - gap: 1ex; -} - -.header { - width: 100%; - display: flex; - flex-direction: row; - flex-wrap: wrap; - vertical-align: middle; -} - -.header div { - padding: 1ex 1em; -} - -.header legend { - display: flex; - flex-direction: row; - gap: 1em; - padding: 0; -} - -.header .note { - width: 100%; - flex-basis: content; - flex-grow: 1; -} - -.header .webtreemap-node { - flex-basis: max-content; - flex-grow: 0; - position: relative; - text-align: center; -} - -#map { - flex-basis: 100%; - - min-width: 300px; - min-height: 300px; - - position: relative; - cursor: pointer; - user-select: none; -} diff --git a/ydb/ci/build_bloat/html_template_bloat/index.html b/ydb/ci/build_bloat/html_template_bloat/index.html deleted file mode 100644 index 9b33a8a73083..000000000000 --- a/ydb/ci/build_bloat/html_template_bloat/index.html +++ /dev/null @@ -1,33 +0,0 @@ - - - - Build bloat - - - - - -
-
-
Click on a box to zoom in. Click on the outermost box to zoom out.
- -
Legend:
-
Namespace
-
Function
- -
-
-
- - - - - diff --git a/ydb/ci/build_bloat/html_template_bloat/webtreemap.css b/ydb/ci/build_bloat/html_template_bloat/webtreemap.css deleted file mode 100644 index 871794c53e0f..000000000000 --- a/ydb/ci/build_bloat/html_template_bloat/webtreemap.css +++ /dev/null @@ -1,65 +0,0 @@ -.webtreemap-node { - /* Required attributes. */ - position: absolute; - overflow: hidden; /* To hide overlong captions. */ - background: white; /* Nodes must be opaque for zIndex layering. */ - border: solid 1px #555; /* Calculations assume 1px border. */ - - /* Optional: CSS animation. */ - transition: top 0.3s, - left 0.3s, - width 0.3s, - height 0.3s; -} - -/* Optional: highlight nodes on mouseover. */ -.webtreemap-node:hover { - background: #FFFFFF; -} - -/* Optional: Different background colors depending on type. */ -.webtreemap-type-namespace { - background: #66C2A5; -} -.webtreemap-type-function { - background: #FC8D62; -} -.webtreemap-type-dir { - background: #8DA0CB; -} - -#legend > * { - border: solid 1px #444; -} - -/* Optional: Different borders depending on level. */ -/* -.webtreemap-level0 { - border: solid 1px #444; -} -.webtreemap-level1 { - border: solid 1px #666; -} -.webtreemap-level2 { - border: solid 1px #888; -} -.webtreemap-level3 { - border: solid 1px #aaa; -} -.webtreemap-level4 { - border: solid 1px #ccc; -} -*/ - -/* Optional: styling on node captions. */ -.webtreemap-caption { - font-family: sans-serif; - font-size: 11px; - padding: 2px; - text-align: center; -} - -/* Optional: styling on captions on mouse hover. */ -/*.webtreemap-node:hover > .webtreemap-caption { - text-decoration: underline; -}*/ diff --git a/ydb/ci/build_bloat/html_template_bloat/webtreemap.js b/ydb/ci/build_bloat/html_template_bloat/webtreemap.js deleted file mode 100644 index 83acff46b8a4..000000000000 --- a/ydb/ci/build_bloat/html_template_bloat/webtreemap.js +++ /dev/null @@ -1,261 +0,0 @@ -// Based on https://github.com/martine/webtreemap -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Size of border around nodes. -// We could support arbitrary borders using getComputedStyle(), but I am -// skeptical the extra complexity (and performance hit) is worth it. - -;(function() { - var kBorderWidth = 1; - - // Padding around contents. - // TODO: do this with a nested div to allow it to be CSS-styleable. - var kPadding = 2; - - var focused = null; - - function focus(tree) { - focused = tree; - - // Hide all visible siblings of all our ancestors by lowering them. - var level = 0; - var root = tree; - while (root.parent) { - root = root.parent; - level += 1; - for (var i = 0, sibling; sibling = root.children[i]; ++i) { - if (sibling.dom) - sibling.dom.style.zIndex = 0; - } - } - var width = root.dom.offsetWidth; - var height = root.dom.offsetHeight; - // Unhide (raise) and maximize us and our ancestors. - for (var t = tree; t.parent; t = t.parent) { - // Shift off by border so we don't get nested borders. - // TODO: actually make nested borders work (need to adjust width/height). - position(t.dom, -kBorderWidth, -kBorderWidth, width, height); - t.dom.style.zIndex = 1; - } - // And layout into the topmost box. - layout(tree, level, width, height); - } - - function escapeHtml(text) { - 'use strict'; - return text.replace(/[\"&'\/<>]/g, function (a) { - return { - '"': '"', '&': '&', "'": ''', - '/': '/', '<': '<', '>': '>' - }[a]; - }); - } - - function makeDom(tree, level) { - var dom = document.createElement('div'); - dom.style.zIndex = 1; - dom.className = 'webtreemap-node webtreemap-level' + Math.min(level, 4); - if (tree.type) { - dom.className += ' webtreemap-type-' + tree.type; - } - if (tree.children) { - dom.className += ' webtreemap-aggregate'; - } - - /* - if (tree.data['$symbol']) { - dom.className += (' webtreemap-symbol-' + - tree.data['$symbol'].replace(' ', '_')); - } - if (tree.data['$dominant_symbol']) { - dom.className += (' webtreemap-symbol-' + - tree.data['$dominant_symbol'].replace(' ', '_')); - dom.className += (' webtreemap-aggregate'); - } - */ - - dom.onmousedown = function(e) { - if (e.button == 0) { - if (focused && tree == focused && focused.parent) { - focus(focused.parent); - } else { - focus(tree); - } - } - e.stopPropagation(); - return true; - }; - - var caption = document.createElement('div'); - caption.className = 'webtreemap-caption'; - caption.innerHTML = escapeHtml(tree.name); - dom.appendChild(caption); - dom.title = tree.name; - - tree.dom = dom; - return dom; - } - - function position(dom, x, y, width, height) { - // CSS width/height does not include border. - width -= kBorderWidth; - height -= kBorderWidth; - - dom.style.left = x + 'px'; - dom.style.top = y + 'px'; - dom.style.width = Math.max(width, 0) + 'px'; - dom.style.height = Math.max(height, 0) + 'px'; - } - - // Given a list of rectangles |nodes|, the 1-d space available - // |space|, and a starting rectangle index |start|, compute an span of - // rectangles that optimizes a pleasant aspect ratio. - // - // Returns [end, sum], where end is one past the last rectangle and sum is the - // 2-d sum of the rectangles' areas. - function selectSpan(nodes, space, start) { - // Add rectangle one by one, stopping when aspect ratios begin to go - // bad. Result is [start,end) covering the best run for this span. - // http://scholar.google.com/scholar?cluster=5972512107845615474 - var node = nodes[start]; - var rmin = node.size; // Smallest seen child so far. - var rmax = rmin; // Largest child. - var rsum = 0; // Sum of children in this span. - var last_score = 0; // Best score yet found. - for (var end = start; node = nodes[end]; ++end) { - var size = node.size; - if (size < rmin) - rmin = size; - if (size > rmax) - rmax = size; - rsum += size; - - // This formula is from the paper, but you can easily prove to - // yourself it's taking the larger of the x/y aspect ratio or the - // y/x aspect ratio. The additional magic fudge constant of 5 - // makes us prefer wider rectangles to taller ones. - var score = Math.max(5*space*space*rmax / (rsum*rsum), - 1*rsum*rsum / (space*space*rmin)); - if (last_score && score > last_score) { - rsum -= size; // Undo size addition from just above. - break; - } - last_score = score; - } - return [end, rsum]; - } - - function layout(tree, level, width, height) { - if (!('children' in tree)) - return; - - var total = tree.size; - - // XXX why do I need an extra -1/-2 here for width/height to look right? - var x1 = 0, y1 = 0, x2 = width - 1, y2 = height - 2; - x1 += kPadding; y1 += kPadding; - x2 -= kPadding; y2 -= kPadding; - y1 += 14; // XXX get first child height for caption spacing - - var pixels_to_units = Math.sqrt(total / ((x2 - x1) * (y2 - y1))); - - for (var start = 0, child; child = tree.children[start]; ++start) { - if ((x2 - x1 < 20 || y2 - y1 < 10) && !child.important) { - if (child.dom) { - child.dom.style.zIndex = 0; - position(child.dom, -2, -2, 0, 0); - } - continue; - } - - // In theory we can dynamically decide whether to split in x or y based - // on aspect ratio. In practice, changing split direction with this - // layout doesn't look very good. - // var ysplit = (y2 - y1) > (x2 - x1); - var ysplit = true; - - var space; // Space available along layout axis. - if (ysplit) - space = (y2 - y1) * pixels_to_units; - else - space = (x2 - x1) * pixels_to_units; - - var span = selectSpan(tree.children, space, start); - var end = span[0], rsum = span[1]; - - // Now that we've selected a span, lay out rectangles [start,end) in our - // available space. - var x = x1, y = y1; - for (var i = start; i < end; ++i) { - child = tree.children[i]; - if (!child.dom) { - child.parent = tree; - child.dom = makeDom(child, level + 1); - tree.dom.appendChild(child.dom); - } else { - child.dom.style.zIndex = 1; - } - var size = child.size; - var frac = size / rsum; - if (ysplit) { - width = rsum / space; - height = size / width; - } else { - height = rsum / space; - width = size / height; - } - width /= pixels_to_units; - height /= pixels_to_units; - width = Math.round(width); - height = Math.round(height); - position(child.dom, x, y, width, height); - if ('children' in child) { - layout(child, level + 1, width, height); - } - if (ysplit) - y += height; - else - x += width; - } - - // Shrink our available space based on the amount we used. - if (ysplit) - x1 += Math.round((rsum / space) / pixels_to_units); - else - y1 += Math.round((rsum / space) / pixels_to_units); - - // end points one past where we ended, which is where we want to - // begin the next iteration, but subtract one to balance the ++ in - // the loop. - start = end - 1; - } - } - - function renderTreemap(dom, data) { - var style = getComputedStyle(dom, null); - var width = parseInt(style.width); - var height = parseInt(style.height); - if (!data.dom) { - makeDom(data, 0); - dom.appendChild(data.dom); - } - position(data.dom, 0, 0, width, height); - layout(data, 0, width, height); - } - - window.renderTreemap = renderTreemap; - })(window); - \ No newline at end of file diff --git a/ydb/ci/build_bloat/main.py b/ydb/ci/build_bloat/main.py index d91f4e519d06..a7134bf2efd3 100755 --- a/ydb/ci/build_bloat/main.py +++ b/ydb/ci/build_bloat/main.py @@ -4,9 +4,10 @@ import json from functools import partial import os -import shutil from concurrent.futures import ProcessPoolExecutor +from jinja2 import Environment, FileSystemLoader, StrictUndefined + HEADER_COMPILE_TIME_TO_SHOW = 0.5 # sec @@ -408,7 +409,21 @@ def main(): print("Performing '{}'".format(description)) tree = fn(args.build_dir, output_path, base_src_dir) - shutil.copytree(html_dir, output_path, dirs_exist_ok=True) + env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) + types = [ + ("h", "Header", "#66C2A5"), + ("cpp", "Cpp", "#FC8D62"), + ("dir", "Dir", "#8DA0CB"), + ] + file_names = os.listdir(html_dir) + os.makedirs(output_path, exist_ok=True) + for file_name in file_names: + data = env.get_template(file_name).render(types=types) + + dst_path = os.path.join(output_path, file_name) + with open(dst_path, "w") as f: + f.write(data) + with open(os.path.join(output_path, "bloat.json"), "w") as f: f.write("var kTree = ") json.dump(tree, f, indent=4) diff --git a/ydb/ci/build_bloat/template_bloat.py b/ydb/ci/build_bloat/template_bloat.py index 6e93525e03cb..19892a07395a 100755 --- a/ydb/ci/build_bloat/template_bloat.py +++ b/ydb/ci/build_bloat/template_bloat.py @@ -2,9 +2,10 @@ import argparse import json import os -import shutil import sys +from jinja2 import Environment, FileSystemLoader, StrictUndefined + THRESHHOLD_TO_SHOW_ON_TREE_VIEW = 1024*10 def remove_brackets(name, b1, b2): @@ -210,16 +211,27 @@ def main(): with open(output_prefix + ".by_count.txt","w") as f: for p in sorted(items, key=lambda p: p[1][1], reverse=True): print_stat(f, p) - + if options.html_template_bloat: output_dir = options.html_template_bloat current_script_dir = os.path.dirname(os.path.realpath(__file__)) - html_dir = os.path.join(current_script_dir, "html_template_bloat") + html_dir = os.path.join(current_script_dir, "html") tree = build_tree(items) - shutil.copytree(html_dir, output_dir, dirs_exist_ok=True) - + env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) + types = [ + ("namespace", "Namespace", "#66C2A5"), + ("function", "Function", "#FC8D62"), + ] + file_names = os.listdir(html_dir) + os.makedirs(output_dir, exist_ok=True) + for file_name in file_names: + data = env.get_template(file_name).render(types=types) + + dst_path = os.path.join(output_dir, file_name) + with open(dst_path, "w") as f: + f.write(data) with open(os.path.join(output_dir, "bloat.json"), "w") as f: f.write("kTree = ") From eee469d3d8744c0413cd4ee5afd21705b861e298 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 5 Aug 2024 09:01:53 +0000 Subject: [PATCH 2/4] wip --- ydb/ci/build_bloat/template_bloat.py | 86 ++++------------------------ ydb/ci/build_bloat/tree_map.py | 82 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 74 deletions(-) create mode 100755 ydb/ci/build_bloat/tree_map.py diff --git a/ydb/ci/build_bloat/template_bloat.py b/ydb/ci/build_bloat/template_bloat.py index 19892a07395a..335182857fe5 100755 --- a/ydb/ci/build_bloat/template_bloat.py +++ b/ydb/ci/build_bloat/template_bloat.py @@ -4,6 +4,8 @@ import os import sys +import tree_map + from jinja2 import Environment, FileSystemLoader, StrictUndefined THRESHHOLD_TO_SHOW_ON_TREE_VIEW = 1024*10 @@ -89,51 +91,9 @@ def print_stat(f, d): for s in sorted(p[2]): print(" " + s, file=f) - -def add_to_tree(tree, path, value, count): - tree["name"] = path[0] - if "children" not in tree: - tree["children"] = {} - if len(path) == 1: - # paths can be the same, but return value differs - # assert "size" not in tree - if "size" not in tree: - tree["size"] = 0 - tree["size"] += value - tree["type"] = "function" - tree["count"] = count - else: - tree["type"] = "namespace" - if path[1] not in tree["children"]: - tree["children"][path[1]] = {} - add_to_tree(tree["children"][path[1]], path[1:], value, count) - -def children_to_list(tree): - if "children" not in tree: - return - tree["children"] = list(tree["children"].values()) - for child in tree["children"]: - children_to_list(child) - -def propogate_size(tree): - if "size" not in tree: - tree["size"] = 0 - for child in tree.get("children", []): - tree["size"] += propogate_size(child) - return tree["size"] - -def enrich_names_with_sec(tree): - area = 0 - for child_ in tree.get("children", []): - enrich_names_with_sec(child_) - - tree["name"] = tree["name"] + " " + "{:_} KiB".format(int(tree["size"]/1024)) - if "count" in tree: - tree["name"] += ", {} times".format(tree["count"]) - -def build_tree(items): - tree = {} +def get_tree_paths(items): total_size = 0 + paths_to_add = [] for name, (size, count, obj_files, avg, min, max) in items: # we skip small entities to order to make html view usable if size < THRESHHOLD_TO_SHOW_ON_TREE_VIEW: @@ -161,13 +121,12 @@ def build_tree(items): root_name = "root (all function less than {} KiB are ommited)".format(THRESHHOLD_TO_SHOW_ON_TREE_VIEW // 1024) path = [root_name] + path - - add_to_tree(tree, path, size, count) - children_to_list(tree) - propogate_size(tree) - enrich_names_with_sec(tree) - print("Total size =", total_size) - return tree + path_with_info = [[chunk, "namespace", 0] for chunk in path] + path_with_info[-1][1] = "function" + path_with_info[-1][2] = size + path_with_info[-1][0] += ", {} times".format(count) + paths_to_add.append(path_with_info) + return paths_to_add def parse_args(): @@ -194,7 +153,6 @@ def parse_args(): ) return parser.parse_args() - def main(): options = parse_args() json_path = options.bloat_json @@ -214,28 +172,8 @@ def main(): if options.html_template_bloat: output_dir = options.html_template_bloat - current_script_dir = os.path.dirname(os.path.realpath(__file__)) - html_dir = os.path.join(current_script_dir, "html") - - tree = build_tree(items) - - env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) - types = [ - ("namespace", "Namespace", "#66C2A5"), - ("function", "Function", "#FC8D62"), - ] - file_names = os.listdir(html_dir) - os.makedirs(output_dir, exist_ok=True) - for file_name in file_names: - data = env.get_template(file_name).render(types=types) - - dst_path = os.path.join(output_dir, file_name) - with open(dst_path, "w") as f: - f.write(data) - - with open(os.path.join(output_dir, "bloat.json"), "w") as f: - f.write("kTree = ") - json.dump(tree, f, indent=4) + tree_paths = get_tree_paths(items) + tree_map.generate_tree_map_html(output_dir, tree_paths, unit_name="KiB", factor=1.0/1024) return 0 diff --git a/ydb/ci/build_bloat/tree_map.py b/ydb/ci/build_bloat/tree_map.py new file mode 100755 index 000000000000..7b7b53621a79 --- /dev/null +++ b/ydb/ci/build_bloat/tree_map.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import json +import os + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +def add_to_tree(tree, path): + current_name, current_type, current_size = path[0] + tree["name"] = current_name + if "children" not in tree: + tree["children"] = {} + if "size" not in tree: + tree["size"] = 0 + + tree["size"] += current_size + tree["type"] = current_type + + if len(path) == 1: + # paths can be the same, but return value differs + # assert "size" not in tree + pass + else: + next_name = path[1][0] + if next_name not in tree["children"]: + tree["children"][next_name] = {} + add_to_tree(tree["children"][next_name], path[1:]) + +def children_to_list(tree): + if "children" not in tree: + return + tree["children"] = list(tree["children"].values()) + for child in tree["children"]: + children_to_list(child) + +def propogate_size(tree): + for child in tree.get("children", []): + tree["size"] += propogate_size(child) + return tree["size"] + +def enrich_names_with_units(tree, unit_name, factor): + area = 0 + for child_ in tree.get("children", []): + enrich_names_with_units(child_, unit_name, factor) + + tree["name"] = tree["name"] + " " + "{:_} {}".format(int(tree["size"]*factor), unit_name) + if "count" in tree: + tree["name"] += ", {} times".format(tree["count"]) + +def build_tree_map(paths_to_add, unit_name, factor): + tree = {} + for path in paths_to_add: + add_to_tree(tree, path) + children_to_list(tree) + propogate_size(tree) + enrich_names_with_units(tree, unit_name, factor) + return tree + + +def generate_tree_map_html(output_dir, tree_paths, unit_name, factor): + current_script_dir = os.path.dirname(os.path.realpath(__file__)) + html_dir = os.path.join(current_script_dir, "html") + + tree = build_tree_map(tree_paths, unit_name, factor) + + env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) + types = [ + ("namespace", "Namespace", "#66C2A5"), + ("function", "Function", "#FC8D62"), + ] + file_names = os.listdir(html_dir) + os.makedirs(output_dir, exist_ok=True) + for file_name in file_names: + data = env.get_template(file_name).render(types=types) + + dst_path = os.path.join(output_dir, file_name) + with open(dst_path, "w") as f: + f.write(data) + + with open(os.path.join(output_dir, "bloat.json"), "w") as f: + f.write("kTree = ") + json.dump(tree, f, indent=4) From 588f7aafa9d0a95a614e6850785d538c8eb79ffd Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 5 Aug 2024 16:13:29 +0000 Subject: [PATCH 3/4] wip --- ydb/ci/build_bloat/main.py | 218 +++++++++++++-------------- ydb/ci/build_bloat/template_bloat.py | 42 ++++-- ydb/ci/build_bloat/tree_map.py | 45 +++--- 3 files changed, 159 insertions(+), 146 deletions(-) diff --git a/ydb/ci/build_bloat/main.py b/ydb/ci/build_bloat/main.py index a7134bf2efd3..2acb5a1b23ce 100755 --- a/ydb/ci/build_bloat/main.py +++ b/ydb/ci/build_bloat/main.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import argparse +import copy import json from functools import partial import os from concurrent.futures import ProcessPoolExecutor -from jinja2 import Environment, FileSystemLoader, StrictUndefined +import tree_map HEADER_COMPILE_TIME_TO_SHOW = 0.5 # sec @@ -57,43 +58,6 @@ def get_compile_duration_and_cpp_path(time_trace_path: str) -> tuple[float, str, return duration_us / 1e6, cpp_file, time_trace_path -def add_to_tree(chunks: list[tuple[str, str]], value: int, tree: dict) -> None: - tree["name"] = chunks[0][0] - tree["type"] = chunks[0][1] - if len(chunks) == 1: - tree["size"] = value - else: - if "children" not in tree: - tree["children"] = [] - for child_ in tree["children"]: - if child_["name"] == chunks[1][0]: - child = child_ - break - - else: - child = {"name": chunks[1][0]} - tree["children"].append(child) - add_to_tree(chunks[1:], value, child) - - -def propogate_area(tree): - area = 0 - for child_ in tree.get("children", []): - propogate_area(child_) - area += child_["size"] - - if "size" not in tree: - tree["size"] = area - - -def enrich_names_with_sec(tree): - area = 0 - for child_ in tree.get("children", []): - enrich_names_with_sec(child_) - - tree["name"] = tree["name"] + " " + "{:_} ms".format(tree["size"]) - - def build_include_tree(path: str, build_output_dir: str, base_src_dir: str) -> list: with open(path) as f: obj = json.load(f) @@ -110,26 +74,74 @@ def build_include_tree(path: str, build_output_dir: str, base_src_dir: str) -> l include_events.sort(key=lambda event: (event[0], -event[1])) - path_to_time = {} - current_includes_stack = [] # stack - last_time_stamp = None - - result = [] + tree_path_to_sum_duration = {} + current_includes_stack = [] for time_stamp, ev, path, duration in include_events: - if current_includes_stack: - last_path = current_includes_stack[-1] - prev = path_to_time.get(last_path, 0) - path_to_time[last_path] = prev + (time_stamp - last_time_stamp) / 1000 / 1000 - if ev == 1: current_includes_stack.append(sanitize_path(path, base_src_dir)) - if duration > HEADER_COMPILE_TIME_TO_SHOW * 1000 * 1000: - result.append((current_includes_stack[:], duration)) + tree_path = tuple(current_includes_stack) + prev = tree_path_to_sum_duration.get(tree_path, 0) + tree_path_to_sum_duration[tree_path] = prev + duration else: assert current_includes_stack[-1] == sanitize_path(path, base_src_dir) current_includes_stack.pop() - last_time_stamp = time_stamp + + # filter small entities + tree_paths_to_include = set() + result = [] + for tree_path, duration in tree_path_to_sum_duration.items(): + if duration > HEADER_COMPILE_TIME_TO_SHOW * 1000 * 1000: + for i in range(1, len(tree_path) + 1): + tree_paths_to_include.add(tree_path[:i]) + + def add_to_tree(tree, tree_path, duration): + if len(tree_path) == 0: + tree["duration"] += duration + else: + if tree_path[0] not in tree["children"]: + tree["children"][tree_path[0]] = { + "duration": 0, + "children": {}, + } + add_to_tree(tree["children"][tree_path[0]], tree_path[1:], duration) + + tree = {"children": {}, "duration": 0} + for tree_path in tree_paths_to_include: + add_to_tree(tree, tree_path, tree_path_to_sum_duration[tree_path]) + + def print_tree(tree, padding): + for child, child_tree in tree["children"].items(): + print(padding + child, child_tree["duration"]) + print_tree(child_tree, padding + " ") + + # handy for debug + # print_tree(tree,"") + + # subtract children + def subtract_duration(tree): + if len(tree["children"]) == 0: + return tree["duration"] + else: + children_duration = 0 + for child, child_tree in tree["children"].items(): + children_duration += subtract_duration(child_tree) + + tree["duration"] -= children_duration + return tree["duration"] + children_duration + + subtract_duration(tree) + + # collect result + result = [] + + def collect(tree, current_tree_path): + if current_tree_path: + result.append((current_tree_path[:], tree["duration"])) + for child, child_tree in tree["children"].items(): + collect(child_tree, current_tree_path + [child]) + + collect(tree, []) return result @@ -163,14 +175,27 @@ def generate_cpp_bloat(build_output_dir: str, result_dir: str, base_src_dir: str cpp_compilation_times = [] total_compilation_time = 0.0 + tree_paths = [] + for duration, path, time_trace_path in result: splitted = path.split(os.sep) chunks = list(zip(splitted, (len(splitted) - 1) * ["dir"] + ["cpp"])) - add_to_tree(chunks, int(duration * 1000), tree) + chunks = ["/"] + chunks + cpp_tree_path = [[chunk, "dir", 0] for chunk in splitted] + cpp_tree_path[-1][1] = "cpp" + + cpp_tree_path_fixed_duration = copy.deepcopy(cpp_tree_path) + cpp_tree_path_fixed_duration[-1][2] = duration * 1000 + include_tree = build_include_tree(time_trace_path, build_output_dir, base_src_dir) + for inc_path, inc_duration in include_tree: - additional_chunks = list(zip(inc_path, "h" * len(inc_path))) - add_to_tree(chunks + additional_chunks, inc_duration / 1000, tree) + include_tree_path = [[chunk, "h", 0] for chunk in inc_path] + include_tree_path[-1][2] = inc_duration / 1000 + cpp_tree_path_fixed_duration[-1][2] -= include_tree_path[-1][2] + tree_paths.append(cpp_tree_path + include_tree_path) + + tree_paths.append(cpp_tree_path_fixed_duration) print("{} -> {:.2f}s".format(path, duration)) cpp_compilation_times.append( { @@ -179,21 +204,12 @@ def generate_cpp_bloat(build_output_dir: str, result_dir: str, base_src_dir: str } ) total_compilation_time += duration - - os.makedirs(result_dir, exist_ok=True) - - human_readable_output = { - "total_compilation_time": total_compilation_time, - "cpp_compilation_times": cpp_compilation_times, - } - - with open(os.path.join(result_dir, "output.json"), "w") as f: - json.dump(human_readable_output, f, indent=4) - - propogate_area(tree) - enrich_names_with_sec(tree) - - return tree + types = [ + ("h", "Header", "#66C2A5"), + ("cpp", "Cpp", "#FC8D62"), + ("dir", "Dir", "#8DA0CB"), + ] + tree_map.generate_tree_map_html(result_dir, tree_paths, unit_name="ms", factor=1, types=types) def parse_includes(trace_path: str, base_src_dir: str) -> tuple[list[tuple[int, str]], dict]: @@ -310,14 +326,16 @@ def generate_header_bloat(build_output_dir: str, result_dir: str, base_src_dir: tree = {} headers_compile_duration = [] - + tree_paths = [] for duration, cnt, path in result: path_chunks = path.split(os.sep) path_chunks[-1] = path_chunks[-1] + " (total {} times)".format(cnt) - path_chunks_count = len(path_chunks) - chunks = list(zip(path_chunks, (path_chunks_count - 1) * ["dir"] + ["h"])) - add_to_tree(chunks, int(duration * 1000), tree) + tree_path = [[chunk, "dir", 0] for chunk in path_chunks] + tree_path[-1][1] = "h" + tree_path[-1][2] = duration * 1000 print("{} -> {:.2f}s (aggregated {} times)".format(path, duration, cnt)) + if duration > HEADER_COMPILE_TIME_TO_SHOW: + tree_paths.append(tree_path) headers_compile_duration.append( { "path": path, @@ -326,6 +344,13 @@ def generate_header_bloat(build_output_dir: str, result_dir: str, base_src_dir: } ) + types = [ + ("h", "Header", "#66C2A5"), + ("cpp", "Cpp", "#FC8D62"), + ("dir", "Dir", "#8DA0CB"), + ] + tree_map.generate_tree_map_html(result_dir, tree_paths, unit_name="ms", factor=1, types=types) + time_breakdown = {} for path in total_time_breakdown: @@ -352,10 +377,6 @@ def generate_header_bloat(build_output_dir: str, result_dir: str, base_src_dir: with open(os.path.join(result_dir, "output.json"), "w") as f: json.dump(human_readable_output, f, indent=4) - propogate_area(tree) - enrich_names_with_sec(tree) - - return tree def parse_args(): @@ -391,44 +412,19 @@ def parse_args(): def main(): args = parse_args() - actions = [] - - if args.html_dir_cpp: - actions.append(("cpp build time impact", generate_cpp_bloat, args.html_dir_cpp)) - - if args.html_dir_cpp: - actions.append(("header build time impact", generate_header_bloat, args.html_dir_headers)) - current_script_dir = os.path.dirname(os.path.realpath(__file__)) base_src_dir = os.path.normpath(os.path.join(current_script_dir, "../../..")) # check we a in root of source tree assert os.path.isfile(os.path.join(base_src_dir, "AUTHORS")) - html_dir = os.path.join(current_script_dir, "html") - - for description, fn, output_path in actions: - print("Performing '{}'".format(description)) - tree = fn(args.build_dir, output_path, base_src_dir) - - env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) - types = [ - ("h", "Header", "#66C2A5"), - ("cpp", "Cpp", "#FC8D62"), - ("dir", "Dir", "#8DA0CB"), - ] - file_names = os.listdir(html_dir) - os.makedirs(output_path, exist_ok=True) - for file_name in file_names: - data = env.get_template(file_name).render(types=types) - - dst_path = os.path.join(output_path, file_name) - with open(dst_path, "w") as f: - f.write(data) - - with open(os.path.join(output_path, "bloat.json"), "w") as f: - f.write("var kTree = ") - json.dump(tree, f, indent=4) - - print("Done '{}'".format(description)) + + + if args.html_dir_cpp: + generate_cpp_bloat(args.build_dir, args.html_dir_cpp, base_src_dir) + print("Done '{}'".format("cpp build time impact")) + if args.html_dir_headers: + generate_header_bloat(args.build_dir, args.html_dir_headers, base_src_dir) + print("Done '{}'".format("header build time impact")) + if __name__ == "__main__": diff --git a/ydb/ci/build_bloat/template_bloat.py b/ydb/ci/build_bloat/template_bloat.py index 335182857fe5..eb47e8583e7f 100755 --- a/ydb/ci/build_bloat/template_bloat.py +++ b/ydb/ci/build_bloat/template_bloat.py @@ -1,32 +1,46 @@ #!/usr/bin/env python3 import argparse import json -import os import sys import tree_map -from jinja2 import Environment, FileSystemLoader, StrictUndefined - THRESHHOLD_TO_SHOW_ON_TREE_VIEW = 1024*10 def remove_brackets(name, b1, b2): inside_template = 0 - final_name = "" - for c in name: + final_name_builder = [] + pos = 0 + while pos != len(name): + pos_next_b1 = name.find(b1, pos) + pos_next_b2 = name.find(b2, pos) + + pos_next = pos_next_b1 + if pos_next == -1: + pos_next = pos_next_b2 + elif pos_next_b2 != -1 and pos_next_b2 < pos_next: + pos_next = pos_next_b2 + + c = name[pos_next] + if c == b1: inside_template += 1 if inside_template == 1: - final_name += c + final_name_builder.append(name[pos:pos_next]) + elif c == b2: inside_template -= 1 if inside_template == 0: - final_name += c + final_name_builder.append(c) else: - if inside_template: - continue - final_name += c - return final_name + if inside_template == 0: + final_name_builder.append(name[pos:pos_next]) + + if pos_next == -1: + break + pos = pos_next + 1 + + return "".join(final_name_builder) def get_aggregation_key(name): final_name = name @@ -173,7 +187,11 @@ def main(): if options.html_template_bloat: output_dir = options.html_template_bloat tree_paths = get_tree_paths(items) - tree_map.generate_tree_map_html(output_dir, tree_paths, unit_name="KiB", factor=1.0/1024) + types = [ + ("namespace", "Namespace", "#66C2A5"), + ("function", "Function", "#FC8D62"), + ] + tree_map.generate_tree_map_html(output_dir, tree_paths, unit_name="KiB", factor=1.0/1024, types=types) return 0 diff --git a/ydb/ci/build_bloat/tree_map.py b/ydb/ci/build_bloat/tree_map.py index 7b7b53621a79..fc02b1ca6d92 100755 --- a/ydb/ci/build_bloat/tree_map.py +++ b/ydb/ci/build_bloat/tree_map.py @@ -5,7 +5,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined -def add_to_tree(tree, path): +def _add_to_tree(tree, path): current_name, current_type, current_size = path[0] tree["name"] = current_name if "children" not in tree: @@ -24,50 +24,49 @@ def add_to_tree(tree, path): next_name = path[1][0] if next_name not in tree["children"]: tree["children"][next_name] = {} - add_to_tree(tree["children"][next_name], path[1:]) + _add_to_tree(tree["children"][next_name], path[1:]) -def children_to_list(tree): +def _children_to_list(tree): if "children" not in tree: return tree["children"] = list(tree["children"].values()) for child in tree["children"]: - children_to_list(child) + _children_to_list(child) -def propogate_size(tree): +def _propogate_size(tree): for child in tree.get("children", []): - tree["size"] += propogate_size(child) + tree["size"] += _propogate_size(child) return tree["size"] -def enrich_names_with_units(tree, unit_name, factor): - area = 0 +def _intify_size(tree): + for child in tree.get("children", []): + _intify_size(child) + tree["size"] = int(tree["size"]) + +def _enrich_names_with_units(tree, unit_name, factor): for child_ in tree.get("children", []): - enrich_names_with_units(child_, unit_name, factor) + _enrich_names_with_units(child_, unit_name, factor) - tree["name"] = tree["name"] + " " + "{:_} {}".format(int(tree["size"]*factor), unit_name) - if "count" in tree: - tree["name"] += ", {} times".format(tree["count"]) + tree["name"] = tree["name"] + ", {:_} {}".format(int(tree["size"]*factor), unit_name) -def build_tree_map(paths_to_add, unit_name, factor): +def _build_tree_map(paths_to_add, unit_name, factor): tree = {} for path in paths_to_add: - add_to_tree(tree, path) - children_to_list(tree) - propogate_size(tree) - enrich_names_with_units(tree, unit_name, factor) + _add_to_tree(tree, path) + _children_to_list(tree) + _propogate_size(tree) + _intify_size(tree) + _enrich_names_with_units(tree, unit_name, factor) return tree -def generate_tree_map_html(output_dir, tree_paths, unit_name, factor): +def generate_tree_map_html(output_dir: str, tree_paths: list[tuple[str, str, int]], unit_name: str, factor: float, types: list[tuple[str, str, str]]): current_script_dir = os.path.dirname(os.path.realpath(__file__)) html_dir = os.path.join(current_script_dir, "html") - tree = build_tree_map(tree_paths, unit_name, factor) + tree = _build_tree_map(tree_paths, unit_name, factor) env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined) - types = [ - ("namespace", "Namespace", "#66C2A5"), - ("function", "Function", "#FC8D62"), - ] file_names = os.listdir(html_dir) os.makedirs(output_dir, exist_ok=True) for file_name in file_names: From e96a60720e4ef76063d7fdbe30587f6974802a11 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 6 Aug 2024 10:32:22 +0000 Subject: [PATCH 4/4] Fix --- ydb/ci/build_bloat/main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ydb/ci/build_bloat/main.py b/ydb/ci/build_bloat/main.py index 2acb5a1b23ce..58183ea5ecff 100755 --- a/ydb/ci/build_bloat/main.py +++ b/ydb/ci/build_bloat/main.py @@ -211,6 +211,16 @@ def generate_cpp_bloat(build_output_dir: str, result_dir: str, base_src_dir: str ] tree_map.generate_tree_map_html(result_dir, tree_paths, unit_name="ms", factor=1, types=types) + os.makedirs(result_dir, exist_ok=True) + + human_readable_output = { + "total_compilation_time": total_compilation_time, + "cpp_compilation_times": cpp_compilation_times, + } + + with open(os.path.join(result_dir, "output.json"), "w") as f: + json.dump(human_readable_output, f, indent=4) + def parse_includes(trace_path: str, base_src_dir: str) -> tuple[list[tuple[int, str]], dict]: print("Processing includes in {}".format(trace_path))