Skip to content

Commit

Permalink
Merge pull request #113 from mmagnus/dev
Browse files Browse the repository at this point in the history
dev
  • Loading branch information
mmagnus authored Apr 3, 2022
2 parents 49009ca + 0ae5665 commit 9163b98
Show file tree
Hide file tree
Showing 5 changed files with 270 additions and 14 deletions.
8 changes: 6 additions & 2 deletions engine/md_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import codecs
from engine.conf import PATH_TO_MD, SCREENSHOT_INBOX, PATH_TO_IMG, IMG_PREFIX, AI_WRITER, USE_RM_TO_REMOVE_FIGURE
from engine.process_md import right_MD_from_webservices, get_youtube_embeds_insert, remove_image, simply_interal_links
from engine.process_md import right_MD_from_webservices, get_youtube_embeds_insert, remove_image, simply_interal_links, insert_file_into_archive, insert_safari_url
from engine.plugins.insert_image import insert_image_in_md
from engine.plugins.draw_secondary_structure import get_ss
from engine.plugins.ia_writer import edit_syntax_from_ai_writer_to_geekbook
Expand Down Expand Up @@ -54,14 +54,18 @@ def compile(self):
if USE_RM_TO_REMOVE_FIGURE:
self.md, use_rm = remove_image(self.md)

self.md, file_inserted = insert_file_into_archive(self.md)

self.md, changed = insert_safari_url(self.md)

# ai writer
is_edit_synatx_ai = False
if AI_WRITER:
self.md, is_edit_synatx_ai = edit_syntax_from_ai_writer_to_geekbook(self.md, IMG_PREFIX)

# check if anything changed
if any([is_get_ss, is_ii, is_right_MD, is_edit_synatx_ai, yti, use_rm,
is_simply_interal_links]):
is_simply_interal_links, file_inserted, changed]):
return True
else:
return False
Expand Down
38 changes: 34 additions & 4 deletions engine/page.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ class Page(object):
def __init__(self, fn):
"""Init a Page and load the content of MD file into self.md"""
self.fn = fn
self.name = fn.replace('.md', '')
# it catches errors if the file is removed
try:
with codecs.open(PATH_TO_MD + sep + fn, "r", "utf-8") as f:
Expand All @@ -57,6 +58,35 @@ def __init__(self, fn):
# logging.error('file removed ' + self.fn)
self.md = None

def to_pdf(self):
from engine.topdf import topdf
# if negative True is first then I guess negative False overwrites /tmp/images
# bug (yeah, so that's why half of images are negatives and other are OK
# keep this order, of use exe not os.system!
topdf(self, negative=False)
topdf(self, negative=True)
pass

def to_pdf_save(self):
fn = '/Users/magnus/geekbook/to-pdf.txt'

s = set()

try:
for i in open(fn):
if i:
s.add(i.strip())
except FileNotFoundError:
pass

s.add(self.fn)

print(len(s), s)

with open(fn, 'w') as f:
for i in s:
f.write(i + '\n')

def get_html(self):
"""Compile md to get html"""
self.html = markdown.markdown(
Expand Down Expand Up @@ -129,11 +159,12 @@ def export(self, path, add_toc, push, readme):
f.write('__place_for_your_imgs__')

# \!\[.*?\]\(imgs/.*?\)
hits = re.findall('\"/imgs/.*?\"', content, re.M|re.DOTALL)
hits = re.findall('\((imgs/.*?)\)', content, re.M|re.DOTALL)
# print(hits)
for h in hits:
print('Copy ',h.strip())
#print(h)
# print('Copy ',h.strip())
shutil.copy(PATH_TO_MD + h.replace('"',''), path + os.sep + 'imgs')

import subprocess

def exe(cmd):
Expand Down Expand Up @@ -172,7 +203,6 @@ def is_changed(self):
if self.md:
if not os.path.exists(PATH_TO_ORIG):
os.makedirs(PATH_TO_ORIG)

try:
with codecs.open(PATH_TO_ORIG + sep + self.fn, "r", "utf-8") as f:
orig_md = f.read()
Expand Down
67 changes: 67 additions & 0 deletions engine/process_md.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,25 @@

from engine.conf import PATH_TO_IMG, PATH_TO_MD, PATH_TO_ORIG

from icecream import ic
import sys
ic.configureOutput(outputFunction=lambda *a: print(*a, file=sys.stderr), includeContext=True)
ic.configureOutput(prefix='> ')

import logging
logger = logging.getLogger('geekbook')
logger.setLevel(logging.INFO)


import subprocess
def exe(cmd):
o = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out = o.stdout.read().strip().decode()
err = o.stderr.read().strip().decode()
return out, err


def right_MD_from_webservices(text):
""" Just paste the url generated by Dropbox to convert it in a markdown img """
changed = False
Expand Down Expand Up @@ -58,6 +72,55 @@ def simply_interal_links(text):
ntext += l + '\n'
return ntext, changed

def insert_safari_url(text):
ntext = ''
changed = False
for l in text.split('\n'):
if l.startswith('\is'):
with open('/tmp/is.applescript', 'w') as f:
f.write("""tell application "Safari"
tell window 1
tell current tab
return URL
end tell
end tell
end tell""")
url, err = exe('osascript /tmp/is.applescript')
ic(url)
changed = True
l = url
ntext += l + '\n'
return ntext, changed

def insert_file_into_archive(text):
"""Change file:///Users/magnus/Desktop/1VQM-01642-01680_rpr.pdb
into
[file:cwc15-reporters-h5-r6-210803.md]
Fix to deal with this:
[file:cwc15-introduce-bamHI.md#Run-on-gel-210819]
http://127.0.0.1:5000/view/cwc15-introduce-bamHI.html#Run-on-gel-210819
Next: define 127.0.0.1:5000 by option.
"""
ntext = ''
changed = False
GEEKBOOK_ARCHIVE = '/Users/magnus/Dropbox/geekbook-archive/'
for l in text.split('\n'):
if l.startswith('file:///'):
if l.endswith('.png') or l.endswith('.jpeg') or l.endswith('.jpg'):
continue
else:
fpath = l.replace('file://', '')
import shutil
fn = os.path.basename(fpath)
shutil.move(fpath, '' + fn, GEEKBOOK_ARCHIVE)
l = '[ff:' + fn + ']'
changed = True
ntext += l + '\n'
return ntext, changed

def remove_image(text, verbose=False):
"""[#rm] will remove the image"""
changed = False
Expand Down Expand Up @@ -199,3 +262,7 @@ def get_youtube_embeds_insert(text):
changed = True
ntext += l + '\n'
return ntext, changed


if __name__ == '__main__':
print(insert_safari_url('\is'))
135 changes: 135 additions & 0 deletions engine/topdf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
from engine.conf import PATH_TO_MD, PATH_TO_HTML, PATH_TO_ORIG, FIND_FILES_PLUGIN, ADD_EXTRA_SPACE_FOR_NOTES
import os

from icecream import ic
import sys
ic.configureOutput(outputFunction=lambda *a: print(*a, file=sys.stderr))
ic.configureOutput(prefix='')

def topdf(self, negative=True):
if self.name == '_search_':
return
with open(PATH_TO_MD + os.sep + self.fn) as f:
md = f.read()
md = md.replace('.DARK.jpeg', '.LIGHT.jpeg')
# md = md.replace('\n', '\n\n') # \n
md = md.replace('{{TOC}}', '')

# add a link for easy edits
md = "<b>http://127.0.0.1:5000/view/" + self.name + ".html</b>\n\n" + md
# this does not work
#md = "<a href='http://127.0.0.1:5000/view/" + self.name + ".html'>go to note</a>n" + md

md = md.replace("\n![", "\n\n![") # ?

# revert all images dark
import cv2
import numpy as np
# pip install opencv-python
# taken from
# https://github.com/imneonizer/How-to-find-if-an-image-is-bright-or-dark.git
def isbright(image, dim=10, thresh=0.5):
# 0.2 smaller more dark will be dark
# 0.8 almost all is dark!
# Resize image to 10x10
image = cv2.resize(image, (dim, dim))
# Convert color space to LAB format and extract L channel
L, A, B = cv2.split(cv2.cvtColor(image, cv2.COLOR_BGR2LAB))
# Normalize L channel by dividing all pixel values with maximum pixel value
L = L/np.max(L)
# Return True if mean is greater than thresh else False
return np.mean(L) > thresh

import re
imgs = re.findall('\!\[.*\]\(imgs/(.+)\)', md)
print(imgs)
for i in imgs:
impath = PATH_TO_MD + '/imgs/' + i
tpath = '/tmp/' + i
tpath_neg = '/tmp/neg' + i

if negative:
# check brightness
ic(impath)
try:
image = cv2.imread(impath)
if isbright(image, thresh=0.5): # 0.39
tmpi = i
else:
tmpi = 'neg' + i
# or https://note.nkmk.me/en/python-pillow-invert/
cmd = '/opt/homebrew/bin/convert %s -channel RGB -negate %s' % (impath, tpath_neg)#newpath)
ic(cmd)
except:
tmpi = i

if not negative:
impath = PATH_TO_MD + '/imgs/' + i
cmd = 'cp %s /tmp/%s' % (impath, i)#newpath)
print(cmd)
os.system(cmd)
tmpi = i

#/tmp/qr.jpg
#qi = i + ')![](/tmp/qr.jpg)' # here you get qr.jpg)) so you can fix it
# with replace('qr.jpeg))', 'qr.jpeg)')
#md = md.replace('(imgs/' + i, '(/tmp/' + qi)
QR = False
if QR:
import qrcode
from PIL import Image

# open temp picture
img_bg = Image.open(tpath)

qr = qrcode.QRCode(box_size=img_bg.size[0] / 10) # vs 2 1/10?
qr.add_data('http://18.193.7.215:8080/4j6scj6p82zw400/' + i)
print('http://18.193.7.215:8080/4j6scj6p82zw400/' + i)
qr.make()
# original pictures should be send
cmd = 'scp ' + PATH_TO_MD + '/imgs/' + i + ' aws:/home/ubuntu/rna-tools/rna_tools/tools/webserver-engine/qr/' + i
# for test you can keep this off
# os.system(cmd)

img_qr = qr.make_image()
#pos = (img_bg.size[0] - img_qr.size[0], img_bg.size[1] - img_qr.size[1])
pos = (img_bg.size[0] + 100, img_bg.size[1] - img_qr.size[1])
img_bg.paste(img_qr, pos)
img_bg.save(tpath) #impath) #'data/dst/qr_lena.png')

md = md.replace('(imgs/' + i, '(/tmp/' + tmpi + '){ height=400px }!!!!') # { height=350px }
md = md.replace('!!!!)', '\n') # ugly

#md = md.replace('(imgs/', '(' + PATH_TO_MD + '/imgs/')

md += '\n# Notes\n\n'
md += '|\n\n' * 15 + '\n' # add an empty page
tmp = '/tmp/print.md'
#tmp = PATH_TO_MD + sep + '/tmp.md'
with open(tmp, 'w') as f:
f.write(md)
# PATH_TO_MD + sep + self.fn
# cd ' + PATH_TO_MD + ' &&
if not negative:
output = '~/Dropbox/boox-geekbook-color/' + self.name + '-color.pdf '
else:
output = '~/Dropbox/boox/geekbook/' + self.name + '.pdf '

# no toc for snippets
toc = ' --toc '
if self.name == 'snippets':
toc = ''
# cmd
cmd = 'pandoc ' + tmp + ' -o ' + output + toc + ' --metadata=title=' + self.name + ' -V mainfont="Helvetica" --pdf-engine=xelatex -V geometry:"top=3cm, bottom=3cm, left=3cm, right=3cm"' # -N -f gfm
print(cmd)
if 1: # for testing keep this
import subprocess
def exe(cmd):
o = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out = o.stdout.read().strip().decode()
err = o.stderr.read().strip().decode()
return out, err
exe(cmd)
else:
os.system(cmd)
Loading

0 comments on commit 9163b98

Please sign in to comment.