From c63d54fed75f38741ad17645fbcdc92190d41e42 Mon Sep 17 00:00:00 2001 From: Manas Talukdar Date: Sat, 9 Jan 2021 23:35:33 -0800 Subject: [PATCH] Updated for for list of repos to use submodules from lcs repo. #4 --- repositories.md | 22 +++++++++++++----- src/update_repositories_md.py | 44 ++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/repositories.md b/repositories.md index 6339974..795a452 100644 --- a/repositories.md +++ b/repositories.md @@ -1,3 +1,7 @@ + + + + # Repositories ## Website @@ -12,9 +16,15 @@ ### Book -- [Building Microservices - Designing Fine-Grained Systems](https://github.com/computer-science-engineering/notes-book_bmdfgs) -- [Designing Data Intensive Applications](https://github.com/computer-science-engineering/notes-book_ddia) -- [Distributed systems for fun and profit](https://github.com/computer-science-engineering/notes-book_dsffp) -- [I Heart Logs: Event Data, Stream Processing, and Data Integration](https://github.com/computer-science-engineering/notes-book_i-heart-logs_jay-kreps) -- [Production-Ready Microservices](https://github.com/computer-science-engineering/notes-book_prm) -- [Scaling Teams: Strategies for Building Successful Teams and Organizations](https://github.com/computer-science-engineering/notes-book_scaling-teams_grosse_loftesness) +- [Building Microservices - Designing Fine-Grained Systems](https://github.com/computer-science-engineering/building-microservices_designing-fine-grained-systems) +- [Designing Data Intensive Applications](https://github.com/computer-science-engineering/designing-data-intensive-applications) +- [Distributed systems for fun and profit](https://github.com/computer-science-engineering/distributed-systems-for-fun-and-profit) +- [I Heart Logs: Event Data, Stream Processing, and Data Integration](https://github.com/computer-science-engineering/i-heart-logs_event-data-stream-processing-data-integration) +- [Production-Ready Microservices](https://github.com/computer-science-engineering/production-ready-microservices) +- [Scaling Teams: Strategies for Building Successful Teams and Organizations](https://github.com/computer-science-engineering/scaling-teams_strategies-for-building-successful-teams-and-organizations) +- [Building Microservices - Designing Fine-Grained Systems](https://github.com/computer-science-engineering/building-microservices_designing-fine-grained-systems) +- [Designing Data Intensive Applications](https://github.com/computer-science-engineering/designing-data-intensive-applications) +- [Distributed systems for fun and profit](https://github.com/computer-science-engineering/distributed-systems-for-fun-and-profit) +- [I Heart Logs: Event Data, Stream Processing, and Data Integration](https://github.com/computer-science-engineering/i-heart-logs_event-data-stream-processing-data-integration) +- [Production-Ready Microservices](https://github.com/computer-science-engineering/production-ready-microservices) +- [Scaling Teams: Strategies for Building Successful Teams and Organizations](https://github.com/computer-science-engineering/scaling-teams_strategies-for-building-successful-teams-and-organizations) diff --git a/src/update_repositories_md.py b/src/update_repositories_md.py index 3cb7a1f..fb285dd 100644 --- a/src/update_repositories_md.py +++ b/src/update_repositories_md.py @@ -1,9 +1,13 @@ import os import json +from itertools import chain +import git from mdutils.mdutils import MdUtils import numpy as np from itertools import groupby -from operator import itemgetter +from operator import itemgetter, sub + +LCS_REPO_NAME = 'learning-computer-science' def walk_max_depth(top, maxdepth): @@ -22,27 +26,43 @@ def find_files(): result = {} root_dir = "./repositories" cwd = os.getcwd() - path_separator = os.sep #print(os.listdir(root_dir)) for root, dirs, files in walk_max_depth(root_dir, 2): dirs.sort() for file in files: if file.endswith("metadata.json"): - metadatafile = os.path.normpath(os.path.join( - cwd, file)).replace(path_separator, "/") - contents = open(metadatafile) - metadata = json.load(contents) - result[root] = (metadatafile, metadata) + metadatafile = os.path.normpath(os.path.join(cwd, file)) + metadata_file = open(metadatafile) + metadata = json.load(metadata_file) + result[os.path.normpath(root)] = (metadatafile, metadata) return result +def get_submodules(files): + submodules_result = {} + cwd = os.getcwd() + for key, value in files.items(): + repo = git.Repo(key) + for submodule in repo.submodules: + path_to_submodule_part = os.path.normpath( + os.path.join(key, submodule.path)) + path_to_metadata_file = os.path.normpath( + os.path.join(cwd, path_to_submodule_part, 'metadata.json')) + metadata_file = open(path_to_metadata_file) + metadata = json.load(metadata_file) + submodules_result[path_to_submodule_part] = (path_to_metadata_file, + metadata) + return dict( + chain.from_iterable(d.items() for d in (files, submodules_result))) + + def get_data(files): data = [] for key, value in files.items(): data_dict = {} data_dict['type'] = value[1]['type'] data_dict['name'] = value[1]['name'] - local_path_parts = value[0].split('/') + local_path_parts = value[0].split(os.path.sep) repo_name = local_path_parts[-2] data_dict[ 'url'] = f'https://github.com/computer-science-engineering/{repo_name}' @@ -58,10 +78,11 @@ def create_file(files): md_file.new_header(level=1, title='Repositories') grouped_by_type = groupby(data, key=itemgetter('type')) for key, value in grouped_by_type: + value_sorted = sorted(value, key=lambda x: x['name']) md_file.new_header(level=2, title=key) if key == 'Reading': - write_reading_entries(value, md_file) - for item in value: + write_reading_entries(value_sorted, md_file) + for item in value_sorted: write_item(item, md_file) md_file.new_line() md_file.create_md_file() @@ -91,7 +112,8 @@ def get_reading_sub_header(file): def main(): """main method.""" files = find_files() - create_file(files) + files_including_submodules = get_submodules(files) + create_file(files_including_submodules) if __name__ == '__main__':