Async func to get_current_schedules

Closes: #515
Change-Id: I8a8d1f9beec752c4bb84894218afca19f9170407

src/quads/cli/cli.py

@@ -12,19 +12,20 @@
 import yaml
 from jinja2 import Template
 from requests import ConnectionError
+
 from quads.config import Config as conf
 from quads.exceptions import CliException, BaseQuadsException
 from quads.helpers.utils import first_day_month, last_day_month
 from quads.quads_api import QuadsApi as Quads, APIServerException, APIBadRequest
 from quads.server.models import Assignment
 from quads.tools import reports
 from quads.tools.external.jira import Jira, JiraException
-from quads.tools.move_and_rebuild import move_and_rebuild, switch_config
-from quads.tools.make_instackenv_json import main as regen_instack
-from quads.tools.simple_table_web import main as regen_heatmap
-from quads.tools.regenerate_wiki import main as regen_wiki
 from quads.tools.foreman_heal import main as foreman_heal
+from quads.tools.make_instackenv_json import main as regen_instack
+from quads.tools.move_and_rebuild import move_and_rebuild, switch_config
 from quads.tools.notify import main as notify
+from quads.tools.regenerate_wiki import main as regen_wiki
+from quads.tools.simple_table_web import main as regen_heatmap
 from quads.tools.validate_env import main as validate_env
 
 default_move_command = "/opt/quads/quads/tools/move_and_rebuild.py"
@@ -1876,7 +1877,7 @@ def action_regen_instack(self):
             self.logger.info("Regenerated 'ocpinventory' for OpenShift Management.")
 
     def action_regen_heatmap(self):
-        regen_heatmap()
+        asyncio.run(regen_heatmap())
         self.logger.info("Regenerated web table heatmap.")
 
     def action_regen_wiki(self):

src/quads/tools/make_instackenv_json.py

@@ -107,9 +107,7 @@ async def make_env_json(filename):
 
 def main():
     tasks = []
-    loop = asyncio.get_event_loop()
-    if not loop:  # pragma: no cover
-        loop = asyncio.new_event_loop()
+    loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     if Config["openstack_management"]:
         fn = functools.partial(make_env_json, "instackenv")

src/quads/tools/simple_table_generator.py

@@ -1,100 +1,190 @@
 #!/usr/bin/env python3
-
 import argparse
-import os
+import asyncio
 import csv
-from datetime import datetime
+import os
 import random
+from datetime import datetime
+from typing import List
+from urllib import parse as url_parse
 
+import aiohttp
+from aiohttp import BasicAuth
 from jinja2 import Template
+from requests import Response
+
 from quads.config import Config
-from quads.quads_api import QuadsApi
-
-
-def random_color():
-    def rand():
-        return random.randint(100, 255)
-
-    return "#%02X%02X%02X" % (rand(), rand(), rand())
-
-
-def generator(_host_file, _days, _month, _year, _gentime):
-    quads = QuadsApi(Config)
-    if _host_file:
-        with open(_host_file, "r") as f:
-            reader = csv.reader(f)
-            hosts = list(reader)
-    else:
-        filtered_hosts = quads.filter_hosts(data={"retired": False, "broken": False})
-        hosts = sorted(filtered_hosts, key=lambda x: x.name)
-
-    lines = []
-    __days = []
-    non_allocated_count = 0
-    all_samples = []
-    all_samples.extend(range(129296, 129510))
-    all_samples.extend(range(128000, 128252))
-    samples = random.sample(all_samples, 200)
-    exclude = [129401, 129484]
-    emojis = [emoji for emoji in samples if emoji not in exclude]
-    colors = [random_color() for _ in range(100)]
-    colors[0] = "#A9A9A9"
-    for i, host in enumerate(hosts):
-        line = {"hostname": host.name}
+from quads.server.models import Schedule, Host
+
+
+class QuadsApiAsync:
+
+    def __init__(self, config: Config):
+        self.config = config
+        self.base_url = config.API_URL
+
+    async def async_get(self, endpoint: str) -> Response:
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                        os.path.join(self.base_url, endpoint),
+                        auth=BasicAuth(self.config.get("quads_api_username"), self.config.get("quads_api_password")),
+                        timeout=60,
+                        verify_ssl=False
+                ) as response:
+                    result = await response.json()
+        except Exception as ex:
+            result = {}
+        return result
+
+    async def async_get_current_schedules(self, data: dict = None) -> List[Schedule]:
+        if data is None:
+            data = {}
+        endpoint = os.path.join("schedules", "current")
+        url = f"{endpoint}"
+        if data:
+            url_params = url_parse.urlencode(data)
+            url = f"{endpoint}?{url_params}"
+        response = await self.async_get(url)
+        schedules = []
+        for schedule in response:
+            schedules.append(Schedule().from_dict(schedule))
+        return schedules
+
+    async def async_filter_hosts(self, data) -> List[Host]:
+        url_params = url_parse.urlencode(data)
+        response = await self.async_get(f"hosts?{url_params}")
+        hosts = []
+        for host in response:
+            host_obj = Host().from_dict(data=host)
+            hosts.append(host_obj)
+        return hosts
+
+
+class HostGenerate:
+
+    BLOCK_SIZE = 10
+
+    def __init__(self):
+        self.hosts = []
+        self.total_current_schedules = {}
+        self.quads_async = QuadsApiAsync(Config)
+        self.colors = []
+        self.emojis = []
+        self.current_host_schedules = {}
+        self.generate_colors()
+
+    def random_color(self):
+        def rand():
+            return random.randint(100, 255)
+
+        return "#%02X%02X%02X" % (rand(), rand(), rand())
+
+    def generate_colors(self):
+        all_samples = []
+        all_samples.extend(range(129296, 129510))
+        all_samples.extend(range(128000, 128252))
+        samples = random.sample(all_samples, 200)
+        exclude = [129401, 129484]
+        self.emojis = [emoji for emoji in samples if emoji not in exclude]
+        self.colors = [self.random_color() for _ in range(100)]
+        self.colors[0] = "#A9A9A9"
+
+    async def order_current_schedules_by_hostname(self):
+        if not self.total_current_schedules:
+            total_current_schedules = await self.quads_async.async_get_current_schedules()
+            for schedule in total_current_schedules:
+                self.total_current_schedules[schedule.host.name] = schedule
+
+    async def get_current_host_schedules(self, host_name):
+        if not self.total_current_schedules:
+            await self.order_current_schedules_by_hostname()
+        return self.total_current_schedules.get(host_name, None)
+
+    async def process_hosts(self, host, _days, _month, _year):
+        non_allocated_count = 0
         __days = []
+        schedules = await self.get_current_host_schedules(host.name)
+        schedule = schedules
+        chosen_color = schedule.assignment.cloud.name[5:] if schedules else "01"
         for j in range(1, _days + 1):
             cell_date = "%s-%.2d-%.2d 01:00" % (_year, _month, j)
             cell_time = datetime.strptime(cell_date, "%Y-%m-%d %H:%M")
-            datearg_iso = cell_time.isoformat()
-            date_str = ":".join(datearg_iso.split(":")[:-1])
-            payload = {"host": host.name, "date": date_str}
-            schedule = None
-            schedules = quads.get_current_schedules(payload)
-            if schedules:
-                schedule = schedules[0]
-                chosen_color = schedule.assignment.cloud.name[5:]
-            else:
-                non_allocated_count += 1
-                chosen_color = "01"
             _day = {
                 "day": j,
                 "chosen_color": chosen_color,
-                "emoji": "&#%s;" % emojis[int(chosen_color) - 1],
-                "color": colors[int(chosen_color) - 1],
+                "emoji": "&#%s;" % self.emojis[int(chosen_color) - 1],
+                "color": self.colors[int(chosen_color) - 1],
                 "cell_date": cell_date,
                 "cell_time": cell_time,
             }
-
             if schedule:
-                assignment = schedule.assignment
-                _day["display_description"] = assignment.description
-                _day["display_owner"] = assignment.owner
-                _day["display_ticket"] = assignment.ticket
+                schedule_start_date = schedule.start
+                schedule_end_date = schedule.end
+                if schedule_start_date <= cell_time <= schedule_end_date:
+                    assignment = schedule.assignment
+                    _day["display_description"] = assignment.description
+                    _day["display_owner"] = assignment.owner
+                    _day["display_ticket"] = assignment.ticket
+                else:
+                    chosen_color = "01"
+                    _day["chosen_color"] = "01"
+                    _day["color"] = self.colors[int(chosen_color) - 1]
+                    _day["color"] = self.colors[int(chosen_color) - 1]
+            else:
+                non_allocated_count += 1
             __days.append(_day)
-
-        line["days"] = __days
-        lines.append(line)
-
-    total_hosts = len(hosts)
-    total_use = len(quads.get_current_schedules())
-    utilization = 100 - (non_allocated_count * 100 // (_days * total_hosts))
-    utilization_daily = total_use * 100 // total_hosts
-    with open(os.path.join(Config.TEMPLATES_PATH, "simple_table_emoji")) as _file:
-        template = Template(_file.read())
-    content = template.render(
-        gentime=_gentime,
-        _days=_days,
-        lines=lines,
-        utilization=utilization,
-        utilization_daily=utilization_daily,
-        total_use=total_use,
-        total_hosts=total_hosts,
-    )
-
-    return content
-
-
-if __name__ == "__main__":  # pragma: no cover
+        return __days, non_allocated_count
+
+    async def generator(self, _host_file, _days, _month, _year, _gentime):
+        if _host_file:
+            with open(_host_file, "r") as f:
+                reader = csv.reader(f)
+                hosts = list(reader)
+        else:
+            if not self.hosts:
+                filtered_hosts = await self.quads_async.async_filter_hosts(data={"retired": False, "broken": False})
+                self.hosts = sorted(filtered_hosts, key=lambda x: x.name)
+            hosts = self.hosts
+
+        if not self.total_current_schedules:
+            await self.order_current_schedules_by_hostname()
+
+        total_current_schedules = self.total_current_schedules
+        lines = []
+        __days = []
+        non_allocated_count = 0
+        host_blocks = [hosts[i:i + self.BLOCK_SIZE] for i in
+                       range(0, len(hosts), self.BLOCK_SIZE)]
+
+        for host_block in host_blocks:
+            tasks = [self.process_hosts(host, _days, _month, _year) for host in host_block]
+            results = await asyncio.gather(*tasks)
+
+            for host, (days, non_allocated) in zip(host_block, results):
+                lines.append({"hostname": host.name, "days": days})
+                non_allocated_count += non_allocated
+
+        total_hosts = len(hosts)
+        total_use = len(total_current_schedules)
+        utilization = 100 - (non_allocated_count * 100 // (_days * total_hosts))
+        utilization_daily = total_use * 100 // total_hosts
+        with open(os.path.join(Config.TEMPLATES_PATH, "simple_table_emoji")) as _file:
+            template = Template(_file.read())
+        content = template.render(
+            gentime=_gentime,
+            _days=_days,
+            lines=lines,
+            utilization=utilization,
+            utilization_daily=utilization_daily,
+            total_use=total_use,
+            total_hosts=total_hosts,
+        )
+
+        return content
+
+
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a simple HTML table with color depicting resource usage for the month"
     )
@@ -151,4 +241,6 @@ def generator(_host_file, _days, _month, _year, _gentime):
     year = args.year
     gentime = args.gentime
 
-    generator(host_file, days, month, year, gentime)
+    generate = HostGenerate()
+    asyncio.run(generate.generator(host_file, days, month, year, gentime))
+

src/quads/tools/simple_table_web.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
-
+import asyncio
 import calendar
 import os
-
 from datetime import datetime, timedelta
-from quads.tools.simple_table_generator import generator
+
 from quads.config import Config
+from quads.tools.simple_table_generator import HostGenerate
 
 
-def main():
+async def main():
+    generate = HostGenerate()
     months_out = 4
     now = datetime.now()
     dates = [now]
@@ -18,13 +19,17 @@ def main():
         next_date = previous + timedelta(calendar.mdays[(previous.month % 12) + 1])
         dates.append(next_date)
 
-    if not os.path.exists(Config["visual_web_dir"]):  # pragma: no cover
+    if not os.path.exists(Config["visual_web_dir"]):
         os.makedirs(Config["visual_web_dir"])
 
     for _date in dates:
         gen_time = "Allocation Map for %s-%.2d" % (_date.year, _date.month)
-        content = generator(None, calendar.mdays[_date.month], _date.month, _date.year, gen_time)
-        file_path = os.path.join(Config["visual_web_dir"], "%s-%.2d.html" % (_date.year, _date.month))
+        content = await generate.generator(
+            None, calendar.mdays[_date.month], _date.month, _date.year, gen_time
+        )
+        file_path = os.path.join(
+            Config["visual_web_dir"], "%s-%.2d.html" % (_date.year, _date.month)
+        )
         with open(file_path, "wb+") as _file:
             _file.write(content.encode("utf-8"))
         os.chmod(file_path, 0o644)
@@ -36,10 +41,14 @@ def main():
     if os.path.exists(_next):
         os.remove(_next)
 
-    current_path = os.path.join(Config["visual_web_dir"], "%s-%.2d.html" % (dates[0].year, dates[0].month))
+    current_path = os.path.join(
+        Config["visual_web_dir"], "%s-%.2d.html" % (dates[0].year, dates[0].month)
+    )
     os.symlink(current_path, _current)
 
-    next_path = os.path.join(Config["visual_web_dir"], "%s-%.2d.html" % (dates[1].year, dates[1].month))
+    next_path = os.path.join(
+        Config["visual_web_dir"], "%s-%.2d.html" % (dates[1].year, dates[1].month)
+    )
     os.symlink(next_path, _next)
 
     files = [html for html in os.listdir(Config["visual_web_dir"]) if ".html" in html]
@@ -51,9 +60,13 @@ def main():
 
     index_path = os.path.join(Config["visual_web_dir"], "index.html")
     with open(index_path, "w+") as index:
+        print(f'File created on {index_path}')
         for line in lines:
             index.write(line)
 
 
-if __name__ == "__main__":  # pragma: no cover
-    main()
+if __name__ == "__main__":
+    start_time = datetime.now()
+    print(f'Start time: {start_time}')
+    asyncio.run(main())
+    print(f'End time: {datetime.now()}, Total time: {(datetime.now() - start_time).total_seconds()}s')