diff --git a/.CodeLumiaignore b/.CodeLumiaignore index ff296ab..101c9f4 100644 --- a/.CodeLumiaignore +++ b/.CodeLumiaignore @@ -173,3 +173,8 @@ LICENSE *.sqlite *.jpg requirements.txt +LICENSE* +*.zip +environment.yml +*.svg +*.jpeg diff --git a/.SourceSageignore b/.SourceSageignore index a9e13f4..46ed59d 100644 --- a/.SourceSageignore +++ b/.SourceSageignore @@ -29,4 +29,5 @@ build tmp .CodeLumiaignore .ipynb -*.ipynb \ No newline at end of file +*.ipynb +tmp diff --git a/.gitignore b/.gitignore index fbc5630..f046733 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,4 @@ cython_debug/ SourceSageAssets tmp -__CodeLumiai.md \ No newline at end of file +__CodeLumiai.md diff --git a/README.md b/README.md index 691791a..90a9333 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ license: mit

～Learn to Code, Step by Step～ -[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/MakiAi/CodeLumia)[![](https://img.shields.io/github/stars/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/last-commit/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/languages/top/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![GitHub Release](https://img.shields.io/github/v/release/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/MakiAi/CodeLumia)[![](https://img.shields.io/github/stars/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/last-commit/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/languages/top/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![GitHub Release](https://img.shields.io/github/v/release/Sunwood-ai-labs/CodeLumia?sort=date&color=red) +](https://github.com/Sunwood-ai-labs/CodeLumia)

@@ -55,18 +56,25 @@ CodeLumiaを使用して、プロジェクトのドキュメンテーション ## 更新履歴 +### [[2024/04/15] CodeLumia v0.4.0](https://github.com/Sunwood-ai-labs/CodeLumia/releases/tag/v0.4.0) +- プレビュー機能の追加（ファイルツリー、マークダウン、プレーンテキスト） +- リポジトリのスキャンとマークダウンファイルの生成機能を強化 +- ドキュメンテーションの改善とコードの可読性向上 +- ファイル操作とGit操作のモジュールを改善 + ### [[2024/04/13] CodeLumia v0.3.0](https://github.com/Sunwood-ai-labs/CodeLumia/releases/tag/v0.3.0) - アプリケーションの機能強化とドキュメントデザインの更新 - デモGIFの更新 - `.CodeLumiaignore`のフォーマット更新とリファクタリング - READMEとSourceSageDocsの大幅な更新 - GIFファイルのGit LFSでの追跡と誤記修正 -- + ### [[2024/04/13] CodeLumia v0.2.0](https://github.com/Sunwood-ai-labs/CodeLumia/releases/tag/v0.2.0) - コードのリファクタリングによる可読性と保守性の向上 - リポジトリのURLを入力してファイルツリーとコードを表示する機能追加 - ユーザーインターフェースの改善とドキュメンテーションの拡充 -- CI/CDの設定とREADMEファイルの更新--- +- CI/CDの設定とREADMEファイルの更新 +- ### [2024/04/12] CodeLumia v0.1.0 - 初回リリース diff --git a/app.py b/app.py index 5fd7ce1..308d910 100644 --- a/app.py +++ b/app.py @@ -25,32 +25,43 @@ # リポジトリのURLを入力するテキストボックス repo_url = st.text_input("リポジトリのURL:") st.markdown("---") -st.markdown("[Full Text](#full-text)") +# st.markdown("[Full Text](#full-text)") # .gitignoreのパターンを編集するサイドバー st.sidebar.title(".CodeLumiaignore Patterns") -ignore_patterns = st.sidebar.text_area("Enter patterns (one per line):", value="\n".join(ignore_patterns), height=600).split("\n") +ignore_patterns = st.sidebar.text_area("Enter patterns (one per line):", value="\n".join(ignore_patterns), height=300).split("\n") +tmp_dir = st.sidebar.text_input('tmp_dir', '/tmp') # 探索の最大深度を入力するテキストボックス -max_depth = st.sidebar.number_input("探索の最大深度:", min_value=1, value=2, step=1) +max_depth = st.sidebar.number_input("探索の最大深度:", min_value=1, value=1, step=1) +preview_markdown = st.sidebar.checkbox('preview markdown', value=False) +preview_plaintext = st.sidebar.checkbox('preview plaintext', value=False) +preview_tree = st.sidebar.checkbox('preview tree', value=True) -if repo_url: - repo_name = repo_url.split("/")[-1].split(".")[0] - repo_path = clone_repository(repo_url, repo_name) +if st.button("CodeLumia Run ...", type="primary"): + if repo_url: + repo_name = repo_url.split("/")[-1].split(".")[0] + with st.status("Scaning repository...", expanded=True): + st.write("clone repository...") + repo_path = clone_repository(repo_url, repo_name, tmp_dir=tmp_dir) + st.write("get file tree...") + file_tree = get_file_tree(repo_path, ignore_patterns, max_depth) + st.write("create markdown content...") + markdown_content = create_markdown_content(repo_name, file_tree, repo_path, ignore_patterns, max_depth) - file_tree = get_file_tree(repo_path, ignore_patterns, max_depth) - markdown_content = create_markdown_content(repo_name, file_tree, repo_path, ignore_patterns, max_depth) + # マークダウンファイルを保存 + save_markdown_file(repo_name, markdown_content) + if(preview_tree): + st.code(f"{file_tree}") - # マークダウンファイルを保存 - output_name = "/tmp/__CodeLumiai" - save_markdown_file(output_name, markdown_content) + # Streamlitアプリケーションの構築 + if(preview_markdown): + st.markdown(markdown_content, unsafe_allow_html=True) - # Streamlitアプリケーションの構築 - st.markdown(markdown_content, unsafe_allow_html=True) + # ダウンロードリンクの作成 + st.markdown(f'
Download Markdown File
', unsafe_allow_html=True) - # ダウンロードリンクの作成 - st.markdown(f'Download Markdown File', unsafe_allow_html=True) - - st.markdown("---") - st.markdown("# Full Text") - st.code(markdown_content) + st.markdown("---") + if(preview_plaintext): + st.markdown("# Full Text") + st.code(markdown_content) diff --git a/docs/page_front.md b/docs/page_front.md index 45b7f3a..1535f88 100644 --- a/docs/page_front.md +++ b/docs/page_front.md @@ -5,7 +5,12 @@

～Learn to Code, Step by Step～ -[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/OFA-Sys/OFA-Image_Caption)[![](https://img.shields.io/github/stars/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/last-commit/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia)[![](https://img.shields.io/github/languages/top/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/OFA-Sys/OFA-Image_Caption) +[![](https://img.shields.io/github/stars/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![](https://img.shields.io/github/last-commit/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![](https://img.shields.io/github/languages/top/Sunwood-ai-labs/CodeLumia)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![GitHub Release](https://img.shields.io/github/v/release/Sunwood-ai-labs/CodeLumia?sort=date&color=red)](https://github.com/Sunwood-ai-labs/CodeLumia) +[![GitHub Tag](https://img.shields.io/github/v/tag/Sunwood-ai-labs/CodeLumia?color=orange)](https://github.com/Sunwood-ai-labs/CodeLumia)

diff --git a/modules/file_operations.py b/modules/file_operations.py index eeed9a5..5d510c1 100644 --- a/modules/file_operations.py +++ b/modules/file_operations.py @@ -1,3 +1,4 @@ + import os import fnmatch @@ -7,12 +8,17 @@ def get_file_tree(repo_path, ignore_patterns, max_depth): # .gitignoreに一致するディレクトリを無視 dirs[:] = [d for d in dirs if not any(fnmatch.fnmatch(d, pattern) for pattern in ignore_patterns)] - level = root.replace(repo_path, "").count(os.sep) + level = root.replace(repo_path, "/").count(os.sep) + # print(f"------------------------- max_depth : {max_depth}") + # print(f"dirs1:{dirs}") + # print(f"level:{level}") + # print(f"files:{files}") if level > max_depth: continue indent = " " * 4 * (level) file_tree += f"{indent}{os.path.basename(root)}/\n" + subindent = " " * 4 * (level + 1) for f in files: # .gitignoreに一致するファイルを無視 @@ -26,7 +32,7 @@ def process_files(repo_path, ignore_patterns, max_depth): # .gitignoreに一致するディレクトリを無視 dirs[:] = [d for d in dirs if not any(fnmatch.fnmatch(d, pattern) for pattern in ignore_patterns)] - level = root.replace(repo_path, "").count(os.sep) + level = root.replace(repo_path, "/").count(os.sep) if level > max_depth: continue @@ -37,4 +43,19 @@ def process_files(repo_path, ignore_patterns, max_depth): with open(file_path, "r", encoding="utf-8", errors="ignore") as f: content = f.read() file_contents.append((file_path.replace(f'{repo_path}/', ''), content)) - return file_contents \ No newline at end of file + return file_contents + +if __name__ == "__main__": + + repo_path = "tmp/DeepSeek-Math" + # .gitignoreのパターンを読み込む + ignore_patterns = [] + if os.path.exists(".CodeLumiaignore"): + with open(".CodeLumiaignore", "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_patterns.append(line) + max_depth = 1 + file_tree = get_file_tree(repo_path, ignore_patterns, max_depth) + print(file_tree) diff --git a/modules/git_operations.py b/modules/git_operations.py index e3d8b4f..ccbbaa0 100644 --- a/modules/git_operations.py +++ b/modules/git_operations.py @@ -2,21 +2,32 @@ import shutil import time -def clone_repository(repo_url, repo_name): +import os +import shutil +from git import Repo +import time + +def clone_repository(repo_url, repo_name, tmp_dir="./tmp"): # tmpフォルダを削除 - # if os.path.exists("/tmp"): - # shutil.rmtree("/tmp") + # if os.path.exists(tmp_dir): + # shutil.rmtree(tmp_dir) # tmpフォルダを作成 - os.makedirs("/tmp", exist_ok=True) + os.makedirs(tmp_dir, exist_ok=True) # リポジトリのクローン - repo_path = f"/tmp/{repo_name}" + repo_path = os.path.join(tmp_dir, repo_name) if os.path.exists(repo_path): shutil.rmtree(repo_path) - os.system(f"git clone {repo_url} {repo_path}") + Repo.clone_from(repo_url, repo_path) # 一時的な遅延を追加 time.sleep(1) - return repo_path \ No newline at end of file + return repo_path + +if __name__ == "__main__": + repo_url = "https://github.com/deepseek-ai/DeepSeek-Math" + repo_name = repo_url.split("/")[-1].split(".")[0] + tmp_dir = "./tmp" # 必要に応じてtmpディレクトリを指定 + clone_repository(repo_url, repo_name, tmp_dir) \ No newline at end of file