Add support for text summary highlighting

This is a very experimental feature and requires running
summary_highlight_server.py in scripts folder.

pdf_viewer/config.cpp

@@ -56,6 +56,11 @@ extern int SINGLE_MAIN_WINDOW_SIZE[2];
 extern int SINGLE_MAIN_WINDOW_MOVE[2];
 extern float FIT_TO_PAGE_WIDTH_RATIO;
 extern float RULER_PADDING;
+extern std::wstring TEXT_HIGHLIGHT_URL;
+extern bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_REFINE;
+extern bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_FILL;
+extern int TEXT_SUMMARY_CONTEXT_SIZE;
+extern bool USE_HEURISTIC_IF_TEXT_SUMMARY_NOT_AVAILABLE;
 
 template<typename T>
 void* generic_deserializer(std::wstringstream& stream, void* res_) {
@@ -195,8 +200,11 @@ ConfigManager::ConfigManager(const Path& default_path, const std::vector<Path>&
 	configs.push_back({ L"collapsed_toc", &SMALL_TOC, bool_serializer, bool_deserializer });
 	configs.push_back({ L"ruler_mode", &RULER_MODE, bool_serializer, bool_deserializer });
 	configs.push_back({ L"ruler_padding", &RULER_PADDING, float_serializer, float_deserializer });
-
-	//configs.push_back({ L"auto_embed_annotations", &AUTO_EMBED_ANNOTATIONS, bool_serializer, bool_deserializer });
+	configs.push_back({ L"text_summary_url", &TEXT_HIGHLIGHT_URL, string_serializer, string_deserializer });
+	configs.push_back({ L"text_summary_should_refine", &TEXT_SUMMARY_HIGHLIGHT_SHOULD_REFINE, bool_serializer, bool_deserializer });
+	configs.push_back({ L"text_summary_should_fill", &TEXT_SUMMARY_HIGHLIGHT_SHOULD_FILL, bool_serializer, bool_deserializer });
+	configs.push_back({ L"text_summary_context_size", &TEXT_SUMMARY_CONTEXT_SIZE, int_serializer, int_deserializer });
+	configs.push_back({ L"use_heuristic_if_text_summary_not_available", &USE_HEURISTIC_IF_TEXT_SUMMARY_NOT_AVAILABLE, bool_serializer, bool_deserializer });
 
 
 	std::wstring highlight_config_string = L"highlight_color_a";

pdf_viewer/document.cpp

@@ -7,13 +7,19 @@
 #include <map>
 #include <regex>
 #include <qcryptographichash.h>
+#include <qjsondocument.h>
 
 #include <mupdf/pdf.h>
 
 #include "checksum.h"
 
 extern float SMALL_PIXMAP_SCALE;
 extern float HIGHLIGHT_COLORS[26 * 3];
+extern std::wstring TEXT_HIGHLIGHT_URL;
+extern bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_REFINE;
+extern bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_FILL;
+extern int TEXT_SUMMARY_CONTEXT_SIZE;
+extern bool USE_HEURISTIC_IF_TEXT_SUMMARY_NOT_AVAILABLE;
 //extern bool AUTO_EMBED_ANNOTATIONS;
 
 int Document::get_mark_index(char symbol) {
@@ -333,6 +339,17 @@ Document::Document(fz_context* context, std::wstring file_name, DatabaseManager*
 	db_manager(db),
 	checksummer(checksummer){
 	last_update_time = QDateTime::currentDateTime();
+
+	network_access_manager = new QNetworkAccessManager;
+
+    QObject::connect(network_access_manager, &QNetworkAccessManager::finished, [&](QNetworkReply *reply) {
+		QJsonDocument responsedoc = QJsonDocument::fromJson(reply->readAll());
+		std::string page_string = responsedoc.object().value("page").toString().toStdString();
+		int page = atoi(page_string.c_str());
+		std::string highlights = responsedoc.object().value("text").toString().toStdString();
+		cached_fastread_highlights[page] = highlights;
+        reply->deleteLater();
+        });
 }
 
 void Document::count_chapter_pages(std::vector<int> &page_counts) {
@@ -593,6 +610,7 @@ Document::~Document() {
 			std::cerr << "Error: could not drop documnet" << std::endl;
 		}
 	}
+	delete network_access_manager;
 	//this->figure_indexing_thread.join();
 }
 void Document::reload(std::string password) {
@@ -1536,7 +1554,6 @@ void Document::get_text_selection(fz_context* ctx, fz_point selection_begin,
 			}
 		}
 	}
-
 }
 
 void Document::embed_annotations(std::wstring new_file_path) {
@@ -1819,50 +1836,101 @@ bool Document::needs_authentication() {
 }
 
 std::vector<fz_rect> Document::get_highlighted_character_masks(int page) {
+
 	fz_stext_page* stext_page = get_stext_with_page_number(page);
 	std::vector<fz_stext_char*> flat_chars;
 	get_flat_chars_from_stext_page(stext_page, flat_chars);
-
-	std::vector<std::wstring> words;
-	std::vector<std::vector<fz_rect>> word_rects;
-	get_word_rect_list_from_flat_chars(flat_chars, words, word_rects);
-
 	std::vector<fz_rect> res;
 
-	for (int i = 0; i < words.size(); i++) {
+	auto fastread_highlights_ = get_page_fastread_highlights(page);
+	if (fastread_highlights_) {
+		auto fastread_highlights = fastread_highlights_.value();
 
-		std::vector<fz_rect> highlighted_characters;
-
-		if (words[i].size() == 1) {
-			highlighted_characters.push_back(word_rects[i][0]);
-		}
-		else if (words[i].size() == 2) {
-			highlighted_characters.push_back(word_rects[i][0]);
+		if (fastread_highlights.size() != flat_chars.size()) {
+			std::wcout << L"invalid highlight received\n";
+			return res;
 		}
-		else if (words[i].size() == 3) {
-			highlighted_characters.push_back(word_rects[i][0]);
-			highlighted_characters.push_back(word_rects[i][1]);
+
+		std::vector<fz_rect> highlighted_characters;
+		for (int i = 0; i < flat_chars.size(); i++) {
+			if (fastread_highlights[i] == '0') {
+				if (highlighted_characters.size() > 0) {
+					auto word_rects = create_word_rects_multiline(highlighted_characters);
+					for (auto rect : word_rects) {
+						res.push_back(rect);
+					}
+					highlighted_characters.clear();
+				}
+			}
+			else {
+				highlighted_characters.push_back(fz_rect_from_quad(flat_chars[i]->quad));
+			}
 		}
-		else {
-			int num_highlighted = static_cast<int>(words[i].size() * 0.4f);
+		return res;
+	}
+	else {
+		if (!USE_HEURISTIC_IF_TEXT_SUMMARY_NOT_AVAILABLE) return res;
+
+		std::vector<std::wstring> words;
+		std::vector<std::vector<fz_rect>> word_rects;
+		get_word_rect_list_from_flat_chars(flat_chars, words, word_rects);
+
+
+		for (int i = 0; i < words.size(); i++) {
+
+			std::vector<fz_rect> highlighted_characters;
+
+			int num_highlighted = static_cast<int>(std::ceil(words[i].size() * 0.3f));
 			for (int j = 0; j < num_highlighted; j++) {
 				highlighted_characters.push_back(word_rects[i][j]);
 			}
 
+			res.push_back(create_word_rect(highlighted_characters));
 		}
-		res.push_back(create_word_rect(highlighted_characters));
+		return res;
 	}
 
-	//std::vector<fz_rect> res;
+}
 
-	//std::vector<fz_rect> pending_word;
-	//pending_word.push_back(flat_chars[0]);
+std::optional<std::string> Document::get_page_fastread_highlights(int page) {
+	if (cached_fastread_highlights.find(page) != cached_fastread_highlights.end()) {
+		return cached_fastread_highlights[page];
+	}
+	else {
+		cached_fastread_highlights[page] = {};
+		std::vector<fz_stext_char*> chars;
+		auto stext_page = get_stext_with_page_number(page);
+		std::wstring page_text;
+		get_flat_chars_from_stext_page(stext_page, chars);
+		for (auto chr : chars) {
+			page_text.push_back(chr->c);
+		}
 
-	//for (int i = 1; i < flat_chars.size(); i++) {
-	//	if (flat_chars[i - 1]->c == ' ') {
-	//		res.push_back(fz_rect_from_quad(flat_chars[i]->quad));
-	//	}
-	//}
-	return res;
-}
+		const QUrl url = QString::fromStdWString(TEXT_HIGHLIGHT_URL);
+		QNetworkRequest request(url);
+        request.setHeader(QNetworkRequest::ContentTypeHeader,
+            "application/x-www-form-urlencoded");
+        QUrlQuery postData;
+        postData.addQueryItem("text", QString::fromStdWString(page_text));
+        postData.addQueryItem("page", QString::number(page));
 
+		if (TEXT_SUMMARY_HIGHLIGHT_SHOULD_REFINE) {
+			postData.addQueryItem("refine", QString::number(1));
+		}
+		else {
+			postData.addQueryItem("refine", QString::number(0));
+		}
+
+		if (TEXT_SUMMARY_HIGHLIGHT_SHOULD_FILL) {
+			postData.addQueryItem("fill", QString::number(1));
+		}
+		else {
+			postData.addQueryItem("fill", QString::number(0));
+		}
+		postData.addQueryItem("context_size", QString::number(TEXT_SUMMARY_CONTEXT_SIZE));
+
+
+        network_access_manager->post(request, postData.toString(QUrl::FullyEncoded).toUtf8());
+		return {};
+	}
+}

pdf_viewer/document.h

@@ -12,6 +12,9 @@
 #include <qdatetime.h>
 
 #include <qobject.h>
+#include <qnetworkreply.h>
+#include <qjsondocument.h>
+#include <qurlquery.h>
 
 #include <mupdf/fitz.h>
 #include "sqlite3.h"
@@ -33,6 +36,7 @@ class Document {
 	std::vector<TocNode*> top_level_toc_nodes;
 	std::vector<std::wstring> flat_toc_names;
 	std::vector<int> flat_toc_pages;
+	QNetworkAccessManager* network_access_manager = nullptr;
 
 	int page_offset = 0;
 
@@ -42,6 +46,7 @@ class Document {
 
 	std::vector<std::pair<int, fz_stext_page*>> cached_stext_pages;
 	std::vector<std::pair<int, fz_pixmap*>> cached_small_pixmaps;
+	std::map<int, std::optional<std::string>> cached_fastread_highlights;
 
 	fz_context* context = nullptr;
 	std::wstring file_name;
@@ -202,6 +207,7 @@ class Document {
 	bool needs_password();
 	bool needs_authentication();
 	bool apply_password(const char* password);
+	std::optional<std::string> get_page_fastread_highlights(int page);
 	std::vector<fz_rect> get_highlighted_character_masks(int page);
 
 	friend class DocumentManager;

pdf_viewer/main.cpp

@@ -97,6 +97,7 @@ float STATUS_BAR_COLOR[3] = { 0.0f, 0.0f, 0.0f };
 float STATUS_BAR_TEXT_COLOR[3] = { 1.0f, 1.0f, 1.0f };
 std::wstring SEARCH_URLS[26];
 std::wstring EXECUTE_COMMANDS[26];
+std::wstring TEXT_HIGHLIGHT_URL = L"http://localhost:5000/";
 std::wstring MIDDLE_CLICK_SEARCH_ENGINE = L"s";
 std::wstring SHIFT_MIDDLE_CLICK_SEARCH_ENGINE = L"l";
 float HIGHLIGHT_COLORS[26 * 3] = { \
@@ -156,6 +157,10 @@ bool LINEAR_TEXTURE_FILTERING = false;
 bool RULER_MODE = false;
 bool SMALL_TOC = false;
 bool WHEEL_ZOOM_ON_CURSOR = false;
+bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_REFINE = true;
+bool TEXT_SUMMARY_HIGHLIGHT_SHOULD_FILL = true;
+bool USE_HEURISTIC_IF_TEXT_SUMMARY_NOT_AVAILABLE = false;
+int TEXT_SUMMARY_CONTEXT_SIZE = 49;
 float VISUAL_MARK_NEXT_PAGE_FRACTION = 0.25f;
 float VISUAL_MARK_NEXT_PAGE_THRESHOLD = 0.1f;
 float RULER_PADDING = 0.0f;

pdf_viewer/pdf_view_opengl_widget.cpp

@@ -691,13 +691,17 @@ void PdfViewOpenGLWidget::render(QPainter* painter) {
 
 	if (fastread_mode) {
 
-		enable_stencil();
-		write_to_stencil();
 		auto rects = document_view->get_document()->get_highlighted_character_masks(document_view->get_current_page_number());
-		draw_stencil_rects(document_view->get_current_page_number(), rects);
-		use_stencil_to_write();
-		render_transparent_white();
-		disable_stencil();
+
+		if (rects.size() > 0) {
+			enable_stencil();
+			write_to_stencil();
+			draw_stencil_rects(document_view->get_current_page_number(), rects);
+			use_stencil_to_write();
+			render_transparent_background();
+			disable_stencil();
+
+		}
 	}
 
 #ifndef NDEBUG
@@ -1335,7 +1339,7 @@ void PdfViewOpenGLWidget::disable_stencil() {
 	glDisable(GL_STENCIL_TEST);
 }
 
-void PdfViewOpenGLWidget::render_transparent_white() {
+void PdfViewOpenGLWidget::render_transparent_background() {
 
 	float bar_data[] = {
 		-1, -1,
@@ -1347,13 +1351,23 @@ void PdfViewOpenGLWidget::render_transparent_white() {
 	glDisable(GL_CULL_FACE);
 	glUseProgram(shared_gl_objects.vertical_line_program);
 
-	float vertical_line_color[4] = { 1.0f, 1.0f, 1.0f, 0.5f };
+	float background_color[4] = { 1.0f, 1.0f, 1.0f, 0.5f };
 
-	if (vertical_line_color != nullptr) {
-		glUniform4fv(shared_gl_objects.line_color_uniform_location,
-			1,
-			vertical_line_color);
+	if (this->color_mode == ColorPalette::Normal) {
+	}
+	else if (this->color_mode == ColorPalette::Dark) {
+		background_color[0] = background_color[1] = background_color[2] = 0;
+	}
+	else {
+		background_color[0] = CUSTOM_BACKGROUND_COLOR[0];
+		background_color[1] = CUSTOM_BACKGROUND_COLOR[1];
+		background_color[2] = CUSTOM_BACKGROUND_COLOR[2];
 	}
+
+	glUniform4fv(shared_gl_objects.line_color_uniform_location,
+		1,
+		background_color);
+
 	float time = -QDateTime::currentDateTime().msecsTo(creation_time);
 	glUniform1f(shared_gl_objects.line_time_uniform_location, time);
 

pdf_viewer/pdf_view_opengl_widget.h

@@ -163,7 +163,7 @@ class PdfViewOpenGLWidget : public QOpenGLWidget, protected QOpenGLExtraFunction
 	void use_stencil_to_write();
 	void disable_stencil();
 
-	void render_transparent_white();
+	void render_transparent_background();
 
 public:
 

pdf_viewer/utils.cpp

@@ -404,6 +404,29 @@ fz_rect create_word_rect(const std::vector<fz_rect>& chars) {
 	return res;
 }
 
+std::vector<fz_rect> create_word_rects_multiline(const std::vector<fz_rect>& chars) {
+	std::vector<fz_rect> res;
+	std::vector<fz_rect> current_line_chars;
+
+	if (chars.size() == 0) return res;
+	current_line_chars.push_back(chars[0]);
+
+	for (int i = 1; i < chars.size(); i++) {
+		if (chars[i].x0 < chars[i - 1].x0) { // a new line has begun
+			res.push_back(create_word_rect(current_line_chars));
+			current_line_chars.clear();
+			current_line_chars.push_back(chars[i]);
+		}
+		else {
+			current_line_chars.push_back(chars[i]);
+		}
+	}
+	if (current_line_chars.size() > 0) {
+		res.push_back(create_word_rect(current_line_chars));
+	}
+	return res;
+}
+
 fz_rect create_word_rect(const std::vector<fz_stext_char*>& chars) {
 	fz_rect res;
 	res.x0 = res.x1 = res.y0 = res.y1 = 0;
@@ -1497,7 +1520,6 @@ float type_name_similarity_score(std::wstring name1, std::wstring name2) {
 	return common_prefix_index;
 }
 
-
 void check_for_updates(QWidget* parent, std::string current_version) {
 
 	QString url = "https://github.com/ahrm/sioyek/releases/latest";

pdf_viewer/utils.h

@@ -154,3 +154,4 @@ int get_index_from_tag(const std::string& tag);
 std::wstring truncate_string(const std::wstring& inp, int size);
 std::wstring get_page_formatted_string(int page);
 fz_rect create_word_rect(const std::vector<fz_rect>& chars);
+std::vector<fz_rect> create_word_rects_multiline(const std::vector<fz_rect>& chars);