Merge branch 'save_upload_using_thread'

dev_src/_fs_utils.py

@@ -15,9 +15,12 @@
 > Thanks for your help!
 """
 
+from io import BufferedWriter
 import os
 from queue import Queue
 import re
+import time
+import traceback
 import urllib.parse
 
 
@@ -476,3 +479,104 @@ def write(location):
 
 	write(location)
 
+
+class UploadHandler:
+	def __init__(self, uid):
+		self.serial_io = Queue()
+		# format [[temp_file_obj, mode, data?], ...]
+		# if mode is 's' then data is [os_f_path, overwrite]
+		# if mode is 'w' then data is the data to write
+		self.active = True
+		self.waited = 0
+		self.done = False
+		self.uid = uid
+		self.error = False
+		self.nap_time = 1
+
+		self.stop_on_error = True
+
+	def upload(self, temp_file, mode, data=None):
+		"""
+		* format `[[temp_file_obj, mode, data?], ...]`
+		* if `mode` is `s` then data is [os_f_path, overwrite]
+		* if `mode` is `w` then binary data is the data to write
+		"""
+		if not self.done:
+			self.serial_io.put([temp_file, mode, data])
+
+	def start(self, server):
+		try:
+			self._start(server)
+		except Exception as e:
+			traceback.print_exc()
+			server.log_error("Upload Failed")
+			self.kill()
+
+	def err(self, error_msg):
+		self.error = error_msg
+		if self.stop_on_error:
+			self.kill()
+
+	def sleep(self):
+		time.sleep(self.nap_time)
+
+
+
+	def _start(self, server):
+		while self.active or not self.serial_io.empty():
+			if not self.serial_io.empty():
+				self.waited = 0
+				req_data = self.serial_io.get()
+				file:BufferedWriter = req_data[0]
+				mode = req_data[1]
+				data = req_data[2]
+				if mode == "w": # write
+					file.write(data)
+				if mode == "s": #save
+					temp_fn = file.name
+					if not file.closed:
+						file.close()
+						os_f_path, overwrite = data
+						os_fn = os.path.basename(os_f_path)
+
+						name, ext = os.path.splitext(os_f_path)
+						while (not overwrite) and os.path.isfile(os_f_path):
+							n = 1
+							os_f_path = f"{name}({n}){ext}"
+							n += 1
+
+
+						try:
+							if os.path.exists(os_f_path): # if overwrite is disabled then the previous part will handle the new filename.
+								os.remove(os_f_path)
+							os.rename(temp_fn, os_f_path)
+						except Exception as e:
+							server.log_error(f'Failed to replace {temp_fn} with {os_f_path} by {self.uid}')
+							server.log_error(traceback.format_exc())
+							self.err(f"Failed to upload {os_fn}")
+
+							break
+
+			else:
+				self.waited += 1
+				self.sleep()
+				if self.waited > 100:
+					self.active = False
+					break
+
+	def kill(self):
+		self.active = False
+		self.done = True
+
+		for f in tuple(self.serial_io):
+			name = f[0].name
+			if not f[0].closed:
+				f[0].close()
+
+			if os.path.exists(name):
+				os.remove(name)
+
+		self.serial_io.queue.clear()
+
+
+

dev_src/local_server_pyrobox.py

@@ -14,6 +14,7 @@
 import datetime
 
 
+import threading
 import urllib.parse
 import urllib.request
 
@@ -27,7 +28,7 @@
 
 from pyroboxCore import config as CoreConfig, logger, DealPostData as DPD, run as run_server, tools, reload_server, __version__
 
-from _fs_utils import get_titles, dir_navigator, get_dir_size, get_stat, get_tree_count_n_size, fmbytes, humanbytes
+from _fs_utils import get_titles, dir_navigator, get_dir_size, get_stat, get_tree_count_n_size, fmbytes, humanbytes, UploadHandler
 from _arg_parser import main as arg_parser
 import _page_templates as pt
 from _exceptions import LimitExceed
@@ -1144,7 +1145,22 @@ def upload(self: SH, *args, **kwargs):
 		return None
 
 
-	uploaded_files = [] # Uploaded folder list
+	# uploaded_files = [] # Uploaded folder list
+
+	upload_handler = UploadHandler(uid)
+
+	upload_thread = threading.Thread(target=upload_handler.start, args=(self,))
+	upload_thread.start()
+
+	def remove_from_temp(temp_fn):
+		try:
+			upload_handler.kill()
+		except OSError:
+			pass
+
+		finally:
+			if temp_fn in CoreConfig.temp_file:
+				CoreConfig.temp_file.remove(temp_fn)
 
 
 
@@ -1184,43 +1200,42 @@ def upload(self: SH, *args, **kwargs):
 
 		# ORIGINAL FILE STARTS FROM HERE
 		try:
-			with open(temp_fn, 'wb') as out:
-				preline = post.get()
-				while post.remainbytes > 0:
-					line = post.get()
-					if post.boundary in line:
+			out = open(temp_fn, 'wb')
+			preline = post.get()
+			while post.remainbytes > 0 and not upload_handler.error:
+				line = post.get()
+				if post.boundary in line:
+					preline = preline[0:-1]
+					if preline.endswith(b'\r'):
 						preline = preline[0:-1]
-						if preline.endswith(b'\r'):
-							preline = preline[0:-1]
-						out.write(preline)
-						uploaded_files.append(rltv_path,)
-						break
-					else:
-						out.write(preline)
-						preline = line
 
+					upload_handler.upload(out, 'w', preline)
+					# out.write(preline)
+					# uploaded_files.append(rltv_path,)
+					break
+				else:
+					upload_handler.upload(out, 'w', preline)
+					# out.write(preline)
+					preline = line
+
+			upload_handler.upload(out, 's', (os_f_path, user.MODIFY))
 
-			while (not user.MODIFY) and os.path.isfile(os_f_path):
-				n = 1
-				name, ext = os.path.splitext(os_f_path)
-				fn = f"{name}({n}){ext}"
-				n += 1
-			os.replace(temp_fn, os_f_path)
+			if upload_handler.error and not upload_handler.active:
+				remove_from_temp(temp_fn)
+				return self.send_error(upload_handler.error, HTTPStatus.INTERNAL_SERVER_ERROR, cookie=cookie)
 
 
 
 		except (IOError, OSError):
 			traceback.print_exc()
-
 			return self.send_txt("Can't create file to write, do you have permission to write?", HTTPStatus.SERVICE_UNAVAILABLE, cookie=cookie)
 
-		finally:
-			try:
-				os.remove(temp_fn)
-				CoreConfig.temp_file.remove(temp_fn)
 
-			except OSError:
-				pass
+	upload_handler.active = False # will take no further inputs
+	upload_thread.join()
+
+	if upload_handler.error:
+		return self.send_error(upload_handler.error, HTTPStatus.INTERNAL_SERVER_ERROR, cookie=cookie)
 
 
 

dev_src/pyroboxCore.py

@@ -1684,6 +1684,7 @@ def get(self, show=F, strip=F, Timeout=10, chunk_size=0):
 		if chunk_size <= 0:
 			chunk_size = self.remainbytes
 
+		waited = 0
 		for _ in range(Timeout*2):
 			if self.is_multipart():
 				line = req.rfile.readline()
@@ -1692,6 +1693,8 @@ def get(self, show=F, strip=F, Timeout=10, chunk_size=0):
 			if line:
 				break
 			time.sleep(.5)
+			waited +=.5
+			print(f"Waited for {waited}s")
 		else:
 			raise ConnectionAbortedError
 

dev_src/script_file_list.js

@@ -98,14 +98,14 @@ class UploadManager {
 		let file_list = byId("content_container")
 		this.drag_pop_open = false;
 
-		file_list.ondragover = (event)=>{
+		file_list.ondragover = async (event)=>{
 			event.preventDefault(); //preventing from default behaviour
 			if(that.drag_pop_open){
 				return;
 			}
 			that.drag_pop_open = true;
 
-			form = upload_man.new()
+			form = await upload_man.new()
 			popup_msg.createPopup("Upload Files", form, true, onclose=()=>{
 				that.drag_pop_open = false;
 			})
@@ -125,7 +125,7 @@ class UploadManager {
 		};
 	}
 
-	new() {
+	async new() {
 		//selecting all required elements
 		let that = this;
 		let index = this.last_index;
@@ -335,7 +335,7 @@ class UploadManager {
 			// const filenames = formData.getAll('files').map(v => v.name).join(', ')
 
 			request.open(e.target.method, e.target.action);
-			request.timeout = 60 * 1000;
+			// request.timeout = 60 * 1000; // in case wifi have no internet, it will stop after 60 seconds
 			request.onreadystatechange = () => {
 				if (request.readyState === XMLHttpRequest.DONE) {
 					msg = `${request.status}: ${request.statusText}`;
@@ -423,44 +423,44 @@ class UploadManager {
 			upload_pop_status.innerText = msg;
 		}
 
-		/**
-		 * Checks if a file is already selected or not.
-		 * @param {File} file - The file to check.
-		 * @returns {number} - Returns the index+1 of the file if it exists, otherwise returns 0.
-		 */
-		function uploader_exist(file) {
-			for (let i = 0; i < selected_files.files.length; i++) {
-				const f = selected_files.files[i];
-				if (f.name == file.name) {
-					return i+1; // 0 is false, so we add 1 to make it true
-				}
-			};
-			return 0; // false;
+		function truncate_file_name(name){
+			// if bigger than 20, truncate, veryvery...last6chars
+			if(name.length > 20){
+				return name.slice(0, 7) + "..." + name.slice(-6);
+			}
+			return name;
 		}
 
-
-		/**
-		 * Adds files to the selected files list and replaces any existing file with the same name.
-		 * @param {FileList} files - The list of files to add.
-		 */
-		function addFiles(files) {
-			var exist = false;
-
+		function remove_duplicates(files) {
 			for (let i = 0; i < files.length; i++) {
+				var selected_fnames = [];
 				const file = files[i];
-				exist = uploader_exist(file);
+
+				let exist = [...selected_files.files].findIndex(f => f.name === file.name);
 
-				if (exist) {
+				if (exist > -1) {
 					// if file already selected,
 					// remove that and replace with
 					// new one, because, when uploading
 					// last file will remain in host server,
 					// so we need to replace it with new one
-					toaster.toast("File already selected", 1500);
+					toaster.toast(truncate_file_name(file.name) + " already selected", 1500);
 					selected_files.items.remove(exist-1);
 				}
 				selected_files.items.add(file);
 			};
+		}
+
+
+		/**
+		 * Adds files to the selected files list and replaces any existing file with the same name.
+		 * @param {FileList} files - The list of files to add.
+		 */
+		function addFiles(files) {
+
+			remove_duplicates(files);
+
+
 			log("selected "+ selected_files.items.length+ " files");
 			uploader_showFiles();
 		}
@@ -644,8 +644,8 @@ class FileManager {
 		popup_msg.open_popup();
 	}
 
-	Show_upload_files() {
-		let form = upload_man.new()
+	async Show_upload_files() {
+		let form = await upload_man.new()
 		popup_msg.createPopup("Upload Files", form);
 		popup_msg.open_popup();
 	}