From c1899eab0ba3bb498fd5a06136c0ca730ef339ed Mon Sep 17 00:00:00 2001 From: Zeeland Date: Sat, 17 Dec 2022 23:49:42 +0800 Subject: [PATCH] first commit --- .gitignore | 111 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 77 ++++++++++++++++++++++++++++++++ adapter.py | 41 +++++++++++++++++ config.yaml | 8 ++++ image_service.py | 39 +++++++++++++++++ main.py | 79 +++++++++++++++++++++++++++++++++ requirements.txt | 3 ++ test.md | 19 ++++++++ test.py | 45 +++++++++++++++++++ yaml_service.py | 26 +++++++++++ 10 files changed, 448 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 adapter.py create mode 100644 config.yaml create mode 100644 image_service.py create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 test.md create mode 100644 test.py create mode 100644 yaml_service.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41a8667 --- /dev/null +++ b/.gitignore @@ -0,0 +1,111 @@ +images/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +_trial_temp +.idea + +# vscode +.vscode diff --git a/README.md b/README.md new file mode 100644 index 0000000..f3a0b69 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# md-img-converter +md-img-converter helps you convert the images address in markdown to the address of the specified graph bed. + +> 因为语雀转markdown的时候图片存在防外链行为，如果想要把转出的markdown发表在其他平台，就需要把md中所有的图片地址改成没有放外链的地址，这样子才可以让别人正常查看。该项目旨在解决这个问题，提供了一个可以批量转换markdown中的图片链接为自己的图床的链接的转换器，并重写成一个新的md文件。您只需要修改yaml配置就可以直接运行。 + +## Feature +- download all images of markdown +- upload image to your graph bed +- convert your markdown image in original url to your graph bed url. +- generate a new markdown by your graph bed url. + +> - 批量下载markdown中所有的图片 +> - 上传markdown的图片到你的图床中 +> - 转换你的markdown图像在原始url到您的图形床url。 +> - 生成一个新的markdown文件，里面的图片链接都来自你的图床。 + +## Attention + +1. You can write your own adapter adapter to support other graph beds +2. The link to the image in markdown must be in the form of a url and can be downloaded from the web + +> 1. 读者可以自己编写一个adapter适配器，来支持其他的图床 +> 2. markdown中的图片链接必须是url形式的,可以从web上下载 + +## Usage +> Attention: Your python version must be 3.6 if you want to use aliyun graph bed of oss2. Because oss2 only supports python 3.6 at most. + +> 注意:如果你想使用oss2的阿里云图床，你的python版本必须是3.6。因为oss2最多只支持python 3.6。 +1. pip + +```sh +pip install -r requirements.txt +``` + +2. config yaml + +Open `config.yaml` and config your parameters. You can test by `test.md` +```yaml +# project root markdown file name +file_path: "test.md" +# default adapter is aliyun oss +adapter: "Aliyun" +# Whether to save the image to local +# (no image will be deleted after executing the command) +save_image: True +# Aliyun oss config +Aliyun: + access_key_id: "your key" + access_key_secret: "your key secret" + bucket_name: "your bucket" + place: "beijing" + +``` +3. run your Application + +The only thing you need to do is just run the following command. +```shell script +python main.py +``` + +4. generate the converted file + +New file name is `yourfilename_converted.md`. + +## TODO +- [ ] support more types of graph bed. +- [ ] support command line +- [ ] support UI operate +- [ ] support pypi more easier to operate + + +## How to write other graph bed? +If you want to develop other graph bed, the only thing you need to do is just implement the adapter like `AliyunApater`. Moreover, you need to config `config.yaml`.That's all you need to do. Actually, I've wrapped it so it's easy to extend. + + +## Contribution +Welcome PRs! If you want to contribute to this project, you can submit pr or issue. I am glad to see more people involved and optimize it. \ No newline at end of file diff --git a/adapter.py b/adapter.py new file mode 100644 index 0000000..0320d23 --- /dev/null +++ b/adapter.py @@ -0,0 +1,41 @@ +import oss2 +import yaml_service + +class Adapter: + """ + You can implement Adapter to realize third-party + graph bed. For more details, you can see AliyunAdapter. + """ + + def upload(self, *args, **kwargs): + pass + + def delete(self, *args, **kwargs): + pass + + def get_url(self, *args, **kwargs): + pass + + +class AliyunApater(Adapter): + + def __init__(self): + """ init some aliyun oss config, you should config in config.yaml """ + config = yaml_service.get_adapter_config() + self.access_key_id = config["access_key_id"] + self.access_key_secret = config["access_key_secret"] + self.bucket_name = config["bucket_name"] + self.place = config["place"] + self.endpoint = f'http://oss-cn-{self.place}.aliyuncs.com' + self.auth = oss2.Auth(self.access_key_id, self.access_key_secret) + self.bucket = oss2.Bucket(self.auth, self.endpoint, self.bucket_name) + + def upload(self, key, file): + self.bucket.put_object(key, file) + + def delete(self, key): + self.bucket.delete_object(key) + + def get_url(self, key): + url = f"http://{self.bucket_name}.oss-cn-{self.place}.aliyuncs.com/{key}" + return url diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..677c8a0 --- /dev/null +++ b/config.yaml @@ -0,0 +1,8 @@ +file_path: "test.md" +adapter: "Aliyun" +save_image: True +Aliyun: + access_key_id: "your key" + access_key_secret: "your key secret" + bucket_name: "your bucket" + place: "beijing" diff --git a/image_service.py b/image_service.py new file mode 100644 index 0000000..94ba5c5 --- /dev/null +++ b/image_service.py @@ -0,0 +1,39 @@ +import os +import time +import requests +import logging +from adapter import * + +logger = logging.getLogger(__name__) + + +""" input old img url return new url of your graph bed """ +def get_new_url(origin_image_url, adapter: Adapter, save_img=False) -> str: + file_path = _download_img(origin_image_url) + with open(file_path, "rb") as f: + key = "typora_img/" + f.name.split(os.getcwd() + "\images\\")[-1] + file_data = f.read() + adapter.upload(key, file_data) + new_url = adapter.get_url(key) + logger.info(f"url: {new_url}") + if not save_img: + os.remove(file_path) + return new_url + +def _download_img(url: str): + """ download image from website and stored in file """ + try: + res = requests.get(url) + nowtime = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) + img_dir = os.getcwd() + '\images\\' + if not os.path.exists(img_dir): + os.mkdir(img_dir) + + file_dir = img_dir + nowtime + '.png' + with open(file_dir, "wb") as f: + f.write(res.content) + logger.info(f"filename: {file_dir} has stored successfully") + return file_dir + except Exception as e: + logger.warning(f"download_img failed, reason: {e}") + return None diff --git a/main.py b/main.py new file mode 100644 index 0000000..370f311 --- /dev/null +++ b/main.py @@ -0,0 +1,79 @@ +import re +import logging + +from adapter import AliyunApater, Adapter +import image_service +import yaml_service + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s') +logger = logging.getLogger(__name__) + + +class Application: + """ + This application Helps you convert the image address + in markdown to the address of the specified graph bed. + + Attention: + 1.All your markdown images must be a website url ranther than a file path. + 2.Now only support aliyun-oss. Welcome PRs to add more supports. + """ + + def __init__(self, file_path=None, adapter: Adapter = AliyunApater()): + """ + cofig markdown file path and adapter + + :file_path: original markdown file path + :adapter: use adapter to upload to your graph bed. You can custom your third-party adapter + """ + self.file_path = yaml_service.get_file_path() if not file_path else file_path + self.adapter = adapter + self.save_img_to_local = yaml_service.get_adapter_config + + def run(self): + """ + input a markdown file, this function will replace img address + attention: markdown image must be a website url ranther than a file path. + """ + ori_data = self._read_md() + pre_data = self._find_img_and_replace(ori_data) + self._write_data(pre_data) + logger.info("task end") + + def _write_data(self, data): + new_file_url = self.file_path.replace(".md","_converted.md") + with open(f"{new_file_url}", "w", encoding="utf-8") as f: + f.write(data) + logger.info("write successfully") + + def _read_md(self) -> str: + """ read markdown file and return markdown data """ + with open(self.file_path, "r", encoding="utf-8") as f: + res = f.read() + logger.info("read file successfully") + return res + + def _find_img_and_replace(self, data) -> str: + """ input original markdown data and replace images address """ + images = list(map(lambda item: item[1], + re.findall(r'(?:!\[(.*?)\]\((.*?)\))|', data))) + + for image in images: + data = self._replace_url(data, image) + return data + + def _replace_url(self, data: str, origin_image_url) -> str: + """ replace single image address """ + new_image_url = image_service.get_new_url(origin_image_url, adapter=self.adapter, save_img=self.save_img_to_local) + data = data.replace(origin_image_url, new_image_url) + return data + + +def main(): + app = Application() + app.run() + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a8b9633 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +oss2 +requests +pyyaml \ No newline at end of file diff --git a/test.md b/test.md new file mode 100644 index 0000000..0d6b6c6 --- /dev/null +++ b/test.md @@ -0,0 +1,19 @@ +## 6.3 md图片地址转换 +以下只支持本地传到图床 + +- [https://github.com/JyHu/useful_script.git](https://github.com/JyHu/useful_script.git) +- [https://github.com/JyHu/useful_script/blob/](https://github.com/JyHu/useful_script/blob/master/Scripts/md%E6%96%87%E4%BB%B6%E5%9B%BE%E7%89%87%E5%9B%BE%E5%BA%8A%E8%BD%AC%E6%8D%A2/%E8%87%AA%E5%8A%A8%E8%BD%AC%E6%8D%A2markdown%E6%96%87%E4%BB%B6%E4%B8%AD%E5%9B%BE%E7%89%87%E5%88%B0%E5%9B%BE%E5%BA%8A.md/) + +罢了，折腾了这么久，又试了试这个，发现也不好用。 +![image.png](https://cdn.nlark.com/yuque/0/2022/png/26910220/1670091709979-52f8c3c4-a00f-4668-a236-29ad2c09d0da.png#averageHue=%23272c34&clientId=ubb991e0d-3414-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=928&id=u4a7a8376&margin=%5Bobject%20Object%5D&name=image.png&originHeight=928&originWidth=1050&originalType=binary&ratio=1&rotation=0&showTitle=false&size=201083&status=done&style=none&taskId=u27493dc0-9d78-4c07-929c-cc946d41409&title=&width=1050) + +最后，还是PigGo最香，提供了快捷键上传，上传完之后直接xxxTODO + +## 6.4 Pycasbin + +在pycasbin看到一个经常参与pycasbin的同行，可以参考一些他的contribution： + +- [https://github.com/Nekotoxin/nekotoxin.github.io/blob/gsoc_2022_summary/GSoC2022-summary.md](https://github.com/Nekotoxin/nekotoxin.github.io/blob/gsoc_2022_summary/GSoC2022-summary.md) + + +![image.png](https://cdn.nlark.com/yuque/0/2022/png/26910220/1670150012015-3a93ec6b-bb27-4ed3-b42f-252a0f70b65c.png#averageHue=%23fcfbf5&clientId=u86ce0a81-ec80-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=936&id=ube9c482c&margin=%5Bobject%20Object%5D&name=image.png&originHeight=936&originWidth=1920&originalType=binary&ratio=1&rotation=0&showTitle=false&size=205691&status=done&style=none&taskId=u6a6825da-aaf4-471c-ad0e-2280c325c66&title=&width=1920) \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..6f7fbd5 --- /dev/null +++ b/test.py @@ -0,0 +1,45 @@ +from unittest import TestCase +from main import * +from image_service import get_new_url, _download_img +import yaml_service + +original_image_url = "https://cdn.nlark.com/yuque/0/2022/png/26910220/1670091709979-52f8c3c4-a00f-4668-a236-29ad2c09d0da.png#averageHue=%23272c34&clientId=ubb991e0d-3414-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=928&id=u4a7a8376&margin=%5Bobject%20Object%5D&name=image.png&originHeight=928&originWidth=1050&originalType=binary&ratio=1&rotation=0&showTitle=false&size=201083&status=done&style=none&taskId=u27493dc0-9d78-4c07-929c-cc946d41409&title=&width=1050" + +class TestFunc(TestCase): + + def test_read_yaml_file_path(self): + res = yaml_service.get_file_path() + self.assertIsNotNone(res) + + def test_read_yaml_adapter_config(self): + config = yaml_service.get_adapter_config() + self.assertIsNotNone(config) + + def test_read_md(self): + app = Application() + res = app._read_md() + self.assertIsNotNone(res) + + def test_download(self): + file_dir = _download_img(original_image_url) + self.assertIsNotNone(file_dir) + + def test_uplolad(self): + aliyun_adapter = AliyunApater() + file_dir = _download_img(original_image_url) + aliyun_adapter.upload(key=file_dir, file=file_dir) + aliyun_adapter.delete(key=file_dir) + + def test_replace_single_url(self): + new_image_url = get_new_url(original_image_url, adapter=AliyunApater()) + self.assertIsNotNone(new_image_url) + + def test_replace_md_single_addr(self): + app = Application() + ori_data = app._read_md() + pre_data = app._replace_url(ori_data, original_image_url) + self.assertNotEqual(pre_data, ori_data) + + def test_write(self): + app = Application() + app._write_data("helloworld") diff --git a/yaml_service.py b/yaml_service.py new file mode 100644 index 0000000..e7e7110 --- /dev/null +++ b/yaml_service.py @@ -0,0 +1,26 @@ +import os +import yaml + + +def yaml_dict() -> dict: + with open("config.yaml", "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def get_file_path() -> str: + path = yaml_dict()["file_path"] + return f"{os.getcwd()}\{path}" + + +def get_save_image() -> str: + return yaml_dict()["save_image"] + + +def get_adapter() -> str: + return yaml_dict()["adapter"] + + +def get_adapter_config() -> dict: + """ It need to append more chooses if add more graph bed """ + if yaml_dict()["adapter"] == "Aliyun": + return yaml_dict()["Aliyun"]