From 42f589ccc307a0f00d569b00838291c30bdfc878 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Wed, 24 Jun 2020 19:31:47 +0530 Subject: [PATCH 1/6] upload.sh: Fix parallel downloads for file conatining special characters --- upload.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upload.sh b/upload.sh index 7f2e6a5..402a2ac 100755 --- a/upload.sh +++ b/upload.sh @@ -966,7 +966,7 @@ _process_arguments() { [[ -f ${TMPFILE}ERROR ]] && rm "${TMPFILE}"ERROR # shellcheck disable=SC2016 - printf "%s\n" "${FILENAMES[@]}" | xargs -n1 -P"${NO_OF_PARALLEL_JOBS_FINAL}" -i bash -c ' + printf "\"%s\"\n" "${FILENAMES[@]}" | xargs -n1 -P"${NO_OF_PARALLEL_JOBS_FINAL}" -i bash -c ' _upload_file "${UPLOAD_METHOD:-create}" "{}" "${ID}" "${ACCESS_TOKEN}" parallel ' 1>| "${TMPFILE}"SUCCESS 2>| "${TMPFILE}"ERROR & @@ -1074,7 +1074,7 @@ _process_arguments() { [[ -f "${TMPFILE}"ERROR ]] && rm "${TMPFILE}"ERROR # shellcheck disable=SC2016 - printf "%s\n" "${FINAL_LIST[@]}" | xargs -n1 -P"${NO_OF_PARALLEL_JOBS_FINAL}" -i bash -c ' + printf "\"%s\"\n" "${FINAL_LIST[@]}" | xargs -n1 -P"${NO_OF_PARALLEL_JOBS_FINAL}" -i bash -c ' LIST="{}" FILETOUPLOAD="${LIST//*"|:_//_:|"}" DIRTOUPLOAD="$(: "|:_//_:|""${FILETOUPLOAD}" && : "${LIST::-${#_}}" && printf "%s\n" "${_//*"|:_//_:|"}")" From 3b4fff505bde16cdab2a741c6ebf37ab816bc346 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Wed, 24 Jun 2020 21:35:14 +0530 Subject: [PATCH 2/6] Speedup final list generation for folder uploads * it could be extremely slow because of the for loop, so lets use xargs * Avoid unneccesary checking for sub folders --- upload.sh | 62 ++++++++++++++++++++++++------------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/upload.sh b/upload.sh index 402a2ac..1229ae8 100755 --- a/upload.sh +++ b/upload.sh @@ -945,6 +945,15 @@ _process_arguments() { NEXTROOTDIRID="${WORKSPACE_FOLDER_ID}" + _print_center "justify" "Processing folder.." "-" + # Do not create empty folders during a recursive upload. Use of find in this section is important. + mapfile -t DIRNAMES <<< "$(find "${INPUT}" -type d -not -empty)" + NO_OF_FOLDERS="${#DIRNAMES[@]}" && NO_OF_SUB_FOLDERS="$((NO_OF_FOLDERS - 1))" + _clear_line 1 + if [[ ${NO_OF_SUB_FOLDERS} = 0 ]]; then + SKIP_SUBDIRS="true" + fi + # Skip the sub folders and find recursively all the files and upload them. if [[ -n ${SKIP_SUBDIRS} ]]; then _print_center "justify" "Indexing files recursively.." "-" @@ -955,7 +964,7 @@ _process_arguments() { "${QUIET:-_print_center}" "justify" "Folder: ${FOLDER_NAME} " "| ${NO_OF_FILES} File(s)" "=" && printf "\n" _print_center "justify" "Creating folder.." "-" ID="$(_create_directory "${INPUT}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" && _clear_line 1 - DIRIDS[1]="${ID}" + DIRIDS="${ID}"$'\n' if [[ -n ${parallel} ]]; then { [[ ${NO_OF_PARALLEL_JOBS} -gt ${NO_OF_FILES} ]] && NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_FILES}"; } || { NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_PARALLEL_JOBS}"; } # Export because xargs cannot access if it is just an internal variable. @@ -1007,28 +1016,16 @@ _process_arguments() { _newline "\n" && EMPTY=1 fi else - _print_center "justify" "Indexing files/sub-folders" " recursively.." "-" - # Do not create empty folders during a recursive upload. Use of find in this section is important. - mapfile -t DIRNAMES <<< "$(find "${INPUT}" -type d -not -empty)" - NO_OF_FOLDERS="${#DIRNAMES[@]}" && NO_OF_SUB_FOLDERS="$((NO_OF_FOLDERS - 1))" - # Create a loop and make folders according to list made above. - if [[ ${NO_OF_SUB_FOLDERS} != 0 ]]; then - _clear_line 1 - _print_center "justify" "${NO_OF_SUB_FOLDERS} Sub-folders found." "=" - fi + _print_center "justify" "${NO_OF_SUB_FOLDERS} Sub-folders found." "=" _print_center "justify" "Indexing files.." "=" mapfile -t FILENAMES <<< "$(find "${INPUT}" -type f)" if [[ -n ${FILENAMES[0]} ]]; then NO_OF_FILES="${#FILENAMES[@]}" for _ in {1..3}; do _clear_line 1; done - if [[ ${NO_OF_SUB_FOLDERS} != 0 ]]; then - "${QUIET:-_print_center}" "justify" "${FOLDER_NAME} " "| ${NO_OF_FILES} File(s) | ${NO_OF_SUB_FOLDERS} Sub-folders" "=" - else - "${QUIET:-_print_center}" "justify" "${FOLDER_NAME} " "| ${NO_OF_FILES} File(s)" "=" - fi + "${QUIET:-_print_center}" "justify" "${FOLDER_NAME} " "| ${NO_OF_FILES} File(s) | ${NO_OF_SUB_FOLDERS} Sub-folders" "=" _newline "\n" _print_center "justify" "Creating Folder(s).." "-" - { [[ ${NO_OF_SUB_FOLDERS} != 0 ]] && _newline "\n"; } || : + _newline "\n" unset status DIRIDS for dir in "${DIRNAMES[@]}"; do @@ -1038,31 +1035,26 @@ _process_arguments() { NEXTROOTDIRID="$(printf "%s\n" "${__temp//"|:_//_:|"${__dir}*/}")" fi NEWDIR="${dir##*/}" - [[ ${NO_OF_SUB_FOLDERS} != 0 ]] && _print_center "justify" "Name: ${NEWDIR}" "-" + _print_center "justify" "Name: ${NEWDIR}" "-" ID="$(_create_directory "${NEWDIR}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" # Store sub-folder directory IDs and it's path for later use. ((status += 1)) - DIRIDS[${status}]="$(printf "%s|:_//_:|%s|:_//_:|\n" "${ID}" "${dir}" && printf "\n")" - if [[ ${NO_OF_SUB_FOLDERS} != 0 ]]; then - for _ in {1..2}; do _clear_line 1; done - _print_center "justify" "Status" ": ${status} / ${NO_OF_FOLDERS}" "=" - fi - done - - if [[ ${NO_OF_SUB_FOLDERS} != 0 ]]; then + DIRIDS+="$(printf "%s|:_//_:|%s|:_//_:|\n" "${ID}" "${dir}")"$'\n' for _ in {1..2}; do _clear_line 1; done - else - _clear_line 1 - fi + _print_center "justify" "Status" ": ${status} / ${NO_OF_FOLDERS}" "=" + done + for _ in {1..2}; do _clear_line 1; done _print_center "justify" "Preparing to upload.." "-" - unset status - for file in "${FILENAMES[@]}"; do + _gen_final_list() { + file="${1}" __rootdir="$(_dirname "${file}")" - ((status += 1)) - FINAL_LIST[${status}]="$(printf "%s\n" "${__rootdir}|:_//_:|$(__temp="$(printf "%s\n" "${DIRIDS[@]}" | grep "|:_//_:|${__rootdir}|:_//_:|")" && - printf "%s\n" "${__temp//"|:_//_:|"${__rootdir}*/}")|:_//_:|${file}")" - done + printf "%s\n" "${__rootdir}|:_//_:|$(__temp="$(grep "|:_//_:|${__rootdir}|:_//_:|" <<< "${DIRIDS}")" && + printf "%s\n" "${__temp//"|:_//_:|"${__rootdir}*/}")|:_//_:|${file}" + } + + export -f _gen_final_list _dirname && export DIRIDS + mapfile -t FINAL_LIST <<< "$(printf "\"%s\"\n" "${FILENAMES[@]}" | xargs -n1 -P"${NO_OF_PARALLEL_JOBS:-$(nproc)}" -i bash -c '_gen_final_list "{}"')" if [[ -n ${parallel} ]]; then { [[ ${NO_OF_PARALLEL_JOBS} -gt ${NO_OF_FILES} ]] && NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_FILES}"; } || { NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_PARALLEL_JOBS}"; } @@ -1132,7 +1124,7 @@ _process_arguments() { fi _print_center "justify" "FolderLink" "${SHARE:-}" "-" _is_terminal && _print_center "normal" "$(printf "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93\n")" " " - _print_center "normal" "$(: "$(read -r firstline <<< "${DIRIDS[1]}" && + _print_center "normal" "$(: "$(read -r firstline <<< "${DIRIDS}" && printf "%s\n" "${firstline/"|:_//_:|"*/}")" && printf "%s\n" "${_/$_/https://drive.google.com/open?id=$_}")" " " fi _newline "\n" From 69fcbb3122ebc702ed905b0d1f394962e98aee7d Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Wed, 24 Jun 2020 22:55:30 +0530 Subject: [PATCH 3/6] Handle almost all api errors and print proper error message redirect all error/warning messages to stderr some misc changes to reduce code size --- google-oauth2.sh | 5 +- upload.sh | 148 ++++++++++++++++++++++++----------------------- 2 files changed, 78 insertions(+), 75 deletions(-) diff --git a/google-oauth2.sh b/google-oauth2.sh index 05829e7..01b9615 100755 --- a/google-oauth2.sh +++ b/google-oauth2.sh @@ -89,11 +89,12 @@ if [[ ${1} = create ]]; then printf "Access Token: %s\n" "${ACCESS_TOKEN}" printf "Refresh Token: %s\n" "${REFRESH_TOKEN}" else - _print_center "normal" "Error: Wrong code given, make sure you copy the exact code." "=" + _print_center "justify" "Error: Something went wrong" ", printing error." 1>&2 + printf "%s\n" "${RESPONSE}" 1>&2 exit 1 fi else - _print_center "justify" "No code provided, run the script and try again." "=" + _print_center "justify" "No code provided, run the script and try again." "=" 1>&2 exit 1 fi elif [[ ${1} = refresh ]]; then diff --git a/upload.sh b/upload.sh index 1229ae8..71c114d 100755 --- a/upload.sh +++ b/upload.sh @@ -60,7 +60,7 @@ _update() { _clear_line 1 bash <(printf "%s\n" "${script}") ${job_string:-} --skip-internet-check else - _print_center "justify" "Error: Cannot download ${job} script." "=" + _print_center "justify" "Error: Cannot download ${job} script." "=" 1>&2 exit 1 fi else @@ -70,7 +70,7 @@ _update() { _clear_line 1 bash <(printf "%s\n" "${script}") ${job_string:-} --skip-internet-check else - _print_center "justify" "Error: Cannot download ${job} script." "=" + _print_center "justify" "Error: Cannot download ${job} script." "=" 1>&2 exit 1 fi fi @@ -146,7 +146,10 @@ _check_existing_file() { "${API_URL}/drive/${API_VERSION}/files?q=${query}&fields=files(id)&supportsAllDrives=true")" id="$(_json_value id 1 1 <<< "${search_response}")" - printf "%s\n" "${id}" + + { [[ -z ${id} ]] && _json_value message 1 1 <<< "${search_response}" 1>&2 && return 1; } || { + printf "%s\n" "${id}" + } } ################################################### @@ -186,7 +189,9 @@ _create_directory() { "${API_URL}/drive/${API_VERSION}/files?fields=id&supportsAllDrives=true")" folder_id="$(_json_value id 1 1 <<< "${create_folder_response}")" fi - printf "%s\n" "${folder_id}" + { [[ -z ${folder_id} ]] && _json_value id 1 1 <<< "${create_folder_response}" 1>&2 && return 1; } || { + printf "%s\n" "${folder_id}" + } } ################################################### @@ -233,11 +238,18 @@ _upload_file() { fi fi + _error_logging() { + "${QUIET:-_print_center}" "justify" "Upload ERROR" ", ${slug} not ${string}." "=" 1>&2 && [[ -z ${parallel} ]] && printf "\n\n\n" 1>&2 + UPLOAD_STATUS="ERROR" && export UPLOAD_STATUS # Send a error status, used in folder uploads. + } + # Set proper variables for overwriting files if [[ ${job} = update ]]; then declare existing_file_id # Check if file actually exists, and create if not. - existing_file_id=$(_check_existing_file "${slug}" "${folder_id}" "${ACCESS_TOKEN}") + existing_file_id="$(_check_existing_file "${slug}" "${folder_id}" "${ACCESS_TOKEN}")" || { + printf "%s\n" "${existing_file_id}" 1>&2 && _error_logging && return 1 + } if [[ -n ${existing_file_id} ]]; then if [[ -n ${SKIP_DUPLICATES} ]]; then SKIP_DUPLICATES_FILE_ID="${existing_file_id}" @@ -323,11 +335,6 @@ _upload_file() { "${QUIET:-_print_center}" "justify" "${slug} " "| ${readable_size} | ${string}" "=" } - _error_logging() { - "${QUIET:-_print_center}" "justify" "Upload link generation ERROR" ", ${slug} not ${string}." "=" 1>&2 && [[ -z ${parallel} ]] && printf "\n\n\n" 1>&2 - UPLOAD_STATUS="ERROR" && export UPLOAD_STATUS # Send a error status, used in folder uploads. - } - # Used for resuming interrupted uploads _log_upload_session() { { [[ ${inputsize} -gt 1000000 ]] && printf "%s\n" "${uploadlink}" >| "${__file}"; } || : @@ -440,7 +447,9 @@ _clone_file() { if [[ ${job} = update ]]; then declare existing_file_id # Check if file actually exists. - existing_file_id=$(_check_existing_file "${name}" "${file_root_id}" "${token}") + existing_file_id=$(_check_existing_file "${name}" "${file_root_id}" "${token}") || { + printf "%s\n" "${existing_file_id}" 1>&2 && return 1 + } if [[ -n ${existing_file_id} ]]; then if [[ -n ${SKIP_DUPLICATES} ]]; then FILE_ID="${existing_file_id}" @@ -521,6 +530,8 @@ _share_id() { declare id="${1}" token="${2}" share_email="${3}" role="reader" type="anyone" declare type share_post_data share_post_data share_response share_id + _print_center "justify" "Sharing.." "-" + if [[ -n ${share_email} ]]; then type="user" share_post_data="{\"role\":\"${role}\",\"type\":\"${type}\",\"emailAddress\":\"${share_email}\"}" @@ -535,9 +546,8 @@ _share_id() { "${API_URL}/drive/${API_VERSION}/files/${id}/permissions")" share_id="$(_json_value id 1 1 <<< "${share_response}")" - if [[ -z "${share_id}" ]]; then - _json_value message 1 1 <<< "${share_response}" && return 1 - fi + _clear_line 1 + { [[ -z "${share_id}" ]] && printf "%s\n" "Error: Cannot Share." 1>&2 && _json_value message 1 1 <<< "${share_response}" 1>&2 && return 1; } || return 0 } ################################################### @@ -662,7 +672,7 @@ _setup_arguments() { ID_INPUT_ARRAY+=("$(_extract_id "${2}")") && shift ;; -S | --share) - SHARE=" (SHARED)" + SHARE="_share_id" EMAIL_REGEX="^([A-Za-z]+[A-Za-z0-9]*\+?((\.|\-|\_)?[A-Za-z]+[A-Za-z0-9]*)*)@(([A-Za-z0-9]+)+((\.|\-|\_)?([A-Za-z0-9]+)+)*)+\.([A-Za-z]{2,})+$" if [[ -n ${1} && ! ${1} =~ ^(\-|\-\-) ]]; then SHARE_EMAIL="${2}" && ! [[ ${SHARE_EMAIL} =~ ${EMAIL_REGEX} ]] && printf "\nError: Provided email address for share option is invalid.\n" && exit 1 @@ -751,11 +761,11 @@ _setup_tempfile() { ################################################### # Check Oauth credentials and create/update config file # Client ID, Client Secret, Refesh Token and Access Token -# Globals: 10 variables, 2 functions +# Globals: 10 variables, 3 functions # Variables - API_URL, API_VERSION, TOKEN URL, # CONFIG, UPDATE_DEFAULT_CONFIG, INFO_PATH, # CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN and ACCESS_TOKEN -# Functions - _update_config and _print_center +# Functions - _update_config, _json_value and _print_center # Arguments: None # Result: read description ################################################### @@ -802,12 +812,13 @@ _check_credentials() { _update_config REFRESH_TOKEN "${REFRESH_TOKEN}" "${CONFIG}" _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" else - printf "Error: Wrong code given, make sure you copy the exact code.\n" + _print_center "justify" "Error: Something went wrong" ", printing error." 1>&2 + printf "%s\n" "${RESPONSE}" 1>&2 exit 1 fi else printf "\n" - _print_center "normal" "No code provided, run the script and try again" " " + _print_center "normal" "No code provided, run the script and try again" " " 1>&2 exit 1 fi fi @@ -819,22 +830,26 @@ _check_credentials() { _get_token_and_update() { RESPONSE="$(curl --compressed -s -X POST --data "client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&refresh_token=${REFRESH_TOKEN}&grant_type=refresh_token" "${TOKEN_URL}")" ACCESS_TOKEN="$(_json_value access_token 1 1 <<< "${RESPONSE}")" - ACCESS_TOKEN_EXPIRY="$(curl --compressed -s "${API_URL}/oauth2/${API_VERSION}/tokeninfo?access_token=${ACCESS_TOKEN}" | _json_value exp 1 1)" - _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" - _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + if [[ -n ${ACCESS_TOKEN} ]]; then + ACCESS_TOKEN_EXPIRY="$(curl --compressed -s "${API_URL}/oauth2/${API_VERSION}/tokeninfo?access_token=${ACCESS_TOKEN}" | _json_value exp 1 1)" + _update_config ACCESS_TOKEN "${ACCESS_TOKEN}" "${CONFIG}" + _update_config ACCESS_TOKEN_EXPIRY "${ACCESS_TOKEN_EXPIRY}" "${CONFIG}" + else + _print_center "justify" "Error: Something went wrong" ", printing error." 1>&2 + printf "%s\n" "${RESPONSE}" 1>&2 + exit 1 + fi } - if [[ -z ${ACCESS_TOKEN} ]]; then - _get_token_and_update - elif [[ ${ACCESS_TOKEN_EXPIRY} -lt "$(printf "%(%s)T\\n" "-1")" ]]; then + if [[ -z ${ACCESS_TOKEN} || ${ACCESS_TOKEN_EXPIRY} -lt "$(printf "%(%s)T\\n" "-1")" ]]; then _get_token_and_update fi } ################################################### # Setup root directory where all file/folders will be uploaded/updated -# Globals: 6 variables, 4 functions +# Globals: 6 variables, 5 functions # Variables - ROOTDIR, ROOT_FOLDER, UPDATE_DEFAULT_ROOTDIR, CONFIG, QUIET, ACCESS_TOKEN -# Functions - _print_center, _drive_info, _extract_id, _update_config +# Functions - _print_center, _drive_info, _extract_id, _update_config, _json_value # Arguments: 1 # ${1} = Positive integer ( amount of time in seconds to sleep ) # Result: read description @@ -848,18 +863,19 @@ _setup_root_dir() { declare json json="$(_drive_info "$(_extract_id "${ROOT_FOLDER}")" "id" "${ACCESS_TOKEN}")" if ! [[ ${json} =~ "\"id\"" ]]; then - { [[ ${json} =~ "File not found" ]] && "${QUIET:-_print_center}" "justify" "Given root folder" " ID/URL invalid." "="; } || { - printf "%s\n" "${json}" + { [[ ${json} =~ "File not found" ]] && "${QUIET:-_print_center}" "justify" "Given root folder" " ID/URL invalid." "=" 1>&2; } || { + _json_value message 1 1 <<< "${json}" 1>&2 } exit 1 fi ROOT_FOLDER="$(_json_value id 1 1 <<< "${json}")" - "${1:-_update_config}" ROOT_FOLDER "${ROOT_FOLDER}" "${CONFIG}" + "${1:-:}" ROOT_FOLDER "${ROOT_FOLDER}" "${CONFIG}" } _update_root_id_name() { ROOT_FOLDER_NAME="$(_drive_info "$(_extract_id "${ROOT_FOLDER}")" "name" "${ACCESS_TOKEN}" | _json_value name)" - "${1:-_update_config}" ROOT_FOLDER_NAME "${ROOT_FOLDER_NAME}" "${CONFIG}" + "${1:-:}" ROOT_FOLDER_NAME "${ROOT_FOLDER_NAME}" "${CONFIG}" } + if [[ -n ${ROOTDIR:-} ]]; then ROOT_FOLDER="${ROOTDIR//[[:space:]]/}" { [[ -n ${ROOT_FOLDER} ]] && _check_root_id "${UPDATE_DEFAULT_ROOTDIR}"; } || : @@ -873,28 +889,30 @@ _setup_root_dir() { _update_config ROOT_FOLDER "${ROOT_FOLDER}" "${CONFIG}" fi fi - if [[ -z ${ROOT_FOLDER_NAME} ]]; then - _update_root_id_name "${UPDATE_DEFAULT_ROOTDIR}" - fi + { [[ -z ${ROOT_FOLDER_NAME} ]] && _update_root_id_name "${UPDATE_DEFAULT_ROOTDIR}"; } || : } ################################################### # Setup Workspace folder # Check if the given folder exists in google drive. # If not then the folder is created in google drive under the configured root folder. -# Globals: 3 variables, 2 functions +# Globals: 3 variables, 3 functions # Variables - FOLDERNAME, ROOT_FOLDER, ACCESS_TOKEN -# Functions - _create_directory, _drive_info +# Functions - _create_directory, _drive_info, _json_value # Arguments: None -# Result: read description +# Result: Read Description ################################################### _setup_workspace() { if [[ -z ${FOLDERNAME} ]]; then WORKSPACE_FOLDER_ID="${ROOT_FOLDER}" WORKSPACE_FOLDER_NAME="${ROOT_FOLDER_NAME}" else - WORKSPACE_FOLDER_ID="$(_create_directory "${FOLDERNAME}" "${ROOT_FOLDER}" "${ACCESS_TOKEN}")" - WORKSPACE_FOLDER_NAME="$(_drive_info "${WORKSPACE_FOLDER_ID}" name "${ACCESS_TOKEN}" | _json_value name 1 1)" + WORKSPACE_FOLDER_ID="$(_create_directory "${FOLDERNAME}" "${ROOT_FOLDER}" "${ACCESS_TOKEN}")" || { + _json_value message 1 1 "${WORKSPACE_FOLDER_ID}" 1>&2 && exit 1 + } + WORKSPACE_FOLDER_NAME="$(_drive_info "${WORKSPACE_FOLDER_ID}" name "${ACCESS_TOKEN}" | _json_value name 1 1)" || { + _json_value message 1 1 "${WORKSPACE_FOLDER_NAME}" 1>&2 && exit 1 + } fi } @@ -922,15 +940,8 @@ _process_arguments() { _upload_file "${UPLOAD_METHOD:-create}" "${INPUT}" "${WORKSPACE_FOLDER_ID}" "${ACCESS_TOKEN}" FILE_ID="${SKIP_DUPLICATES_FILE_ID:-${FILE_ID}}" [[ ${UPLOAD_STATUS} = ERROR ]] && for _ in {1..2}; do _clear_line 1; done && continue - if [[ -n "${SHARE}" ]]; then - _print_center "justify" "Sharing the file.." "-" - if SHARE_MSG="$(_share_id "${FILE_ID}" "${ACCESS_TOKEN}" "${SHARE_EMAIL}")"; then - _clear_line 1 - else - _clear_line 1 && printf "%s\n%s\n" "Error: Cannot Share." "${SHARE_MSG}" - fi - fi - _print_center "justify" "DriveLink" "${SHARE:-}" "-" + "${SHARE:-:}" "${FILE_ID}" "${ACCESS_TOKEN}" "${SHARE_EMAIL}" + _print_center "justify" "DriveLink" "${SHARE:+ (SHARED)}" "-" _is_terminal && _print_center "normal" "$(printf "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93\n")" " " _print_center "normal" "${FILE_LINK}" " " printf "\n" @@ -963,7 +974,10 @@ _process_arguments() { for _ in {1..2}; do _clear_line 1; done "${QUIET:-_print_center}" "justify" "Folder: ${FOLDER_NAME} " "| ${NO_OF_FILES} File(s)" "=" && printf "\n" _print_center "justify" "Creating folder.." "-" - ID="$(_create_directory "${INPUT}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" && _clear_line 1 + ID="$(_create_directory "${INPUT}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" || { + printf "%s\n" "${ID}" 1>&2 && return 1 + } + _clear_line 1 DIRIDS="${ID}"$'\n' if [[ -n ${parallel} ]]; then { [[ ${NO_OF_PARALLEL_JOBS} -gt ${NO_OF_FILES} ]] && NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_FILES}"; } || { NO_OF_PARALLEL_JOBS_FINAL="${NO_OF_PARALLEL_JOBS}"; } @@ -1036,7 +1050,9 @@ _process_arguments() { fi NEWDIR="${dir##*/}" _print_center "justify" "Name: ${NEWDIR}" "-" - ID="$(_create_directory "${NEWDIR}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" + ID="$(_create_directory "${NEWDIR}" "${NEXTROOTDIRID}" "${ACCESS_TOKEN}")" || { + printf "%s\n" "${ID}" 1>&2 && return 1 + } # Store sub-folder directory IDs and it's path for later use. ((status += 1)) DIRIDS+="$(printf "%s|:_//_:|%s|:_//_:|\n" "${ID}" "${dir}")"$'\n' @@ -1114,15 +1130,8 @@ _process_arguments() { [[ -z ${VERBOSE:-${VERBOSE_PROGRESS}} ]] && for _ in {1..2}; do _clear_line 1; done if [[ ${SUCCESS_STATUS} -gt 0 ]]; then - if [[ -n ${SHARE} ]]; then - _print_center "justify" "Sharing the folder.." "-" - if SHARE_MSG="$(_share_id "$(read -r firstline <<< "${DIRIDS[1]}" && printf "%s\n" "${firstline/"|:_//_:|"*/}")" "${ACCESS_TOKEN}" "${SHARE_EMAIL}")"; then - _clear_line 1 - else - _clear_line 1 && printf "%s\n%s\n" "Error: Cannot Share." "${SHARE_MSG}" - fi - fi - _print_center "justify" "FolderLink" "${SHARE:-}" "-" + "${SHARE:-:}" "${FILE_ID}" "${ACCESS_TOKEN}" "${SHARE_EMAIL}" + _print_center "justify" "FolderLink" "${SHARE:+ (SHARED)}" "-" _is_terminal && _print_center "normal" "$(printf "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93\n")" " " _print_center "normal" "$(: "$(read -r firstline <<< "${DIRIDS}" && printf "%s\n" "${firstline/"|:_//_:|"*/}")" && printf "%s\n" "${_/$_/https://drive.google.com/open?id=$_}")" " " @@ -1133,7 +1142,7 @@ _process_arguments() { printf "\n" else for _ in {1..2}; do _clear_line 1; done - "${QUIET:-_print_center}" 'justify' "Empty Folder." "-" + "${QUIET:-_print_center}" 'justify' "Empty Folder." "-" 1>&2 printf "\n" fi fi @@ -1144,12 +1153,12 @@ _process_arguments() { json="$(_drive_info "${gdrive_id}" "name,mimeType,size" "${ACCESS_TOKEN}")" || : code="$(_json_value code 1 1 <<< "${json}")" || : if [[ -z ${code} ]]; then - type="$(_json_value mimeType all all <<< "${json}")" - name="$(_json_value name all all <<< "${json}")" - size="$(_json_value size all all <<< "${json}")" + type="$(_json_value mimeType 1 1 <<< "${json}")" || : + name="$(_json_value name 1 1 <<< "${json}")" || : + size="$(_json_value size 1 1 <<< "${json}")" || : for _ in {1..2}; do _clear_line 1; done if [[ ${type} =~ folder ]]; then - _print_center "justify" "Folder not supported." "=" 1>&2 && continue + _print_center "justify" "Folder not supported." "=" 1>&2 && _newline "\n" 1>&2 && continue ## TODO: Add support to clone folders else _print_center "justify" "Given Input" ": File ID" "=" @@ -1157,15 +1166,8 @@ _process_arguments() { _clone_file "${UPLOAD_METHOD:-create}" "${gdrive_id}" "${WORKSPACE_FOLDER_ID}" "${ACCESS_TOKEN}" "${name}" "${size}" [[ ${UPLOAD_STATUS} = ERROR ]] && for _ in {1..2}; do _clear_line 1; done && continue fi - if [[ -n "${SHARE}" ]]; then - _print_center "justify" "Sharing the file.." "-" - if SHARE_MSG="$(_share_id "${FILE_ID}" "${ACCESS_TOKEN}" "${SHARE_EMAIL}")"; then - _clear_line 1 - else - _clear_line 1 && printf "%s\n%s\n" "Error: Cannot Share." "${SHARE_MSG}" - fi - fi - _print_center "justify" "DriveLink" "${SHARE:-}" "-" + "${SHARE:-:}" "${FILE_ID}" "${ACCESS_TOKEN}" "${SHARE_EMAIL}" + _print_center "justify" "DriveLink" "${SHARE:+ (SHARED)}" "-" _is_terminal && _print_center "normal" "$(printf "\xe2\x86\x93 \xe2\x86\x93 \xe2\x86\x93\n")" " " _print_center "normal" "${FILE_LINK}" " " printf "\n" From 4580147f802caaa26db6758032e49b519e4fa187 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Thu, 25 Jun 2020 08:16:33 +0530 Subject: [PATCH 4/6] upload.sh: Fix directory creation name if current folder is uploaded --- upload.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upload.sh b/upload.sh index 71c114d..2075bd8 100755 --- a/upload.sh +++ b/upload.sh @@ -136,7 +136,7 @@ _drive_info() { ################################################### _check_existing_file() { [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1 - declare name="${1}" rootdir="${2}" token="${3}" + declare name="${1##*/}" rootdir="${2}" token="${3}" declare query search_response id query="$(_url_encode "name='${name}' and '${rootdir}' in parents and trashed=false and 'me' in writers")" @@ -167,7 +167,7 @@ _check_existing_file() { ################################################### _create_directory() { [[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1 - declare dirname="${1}" rootdir="${2}" token="${3}" + declare dirname="${1##*/}" rootdir="${2}" token="${3}" declare query search_response folder_id query="$(_url_encode "mimeType='application/vnd.google-apps.folder' and name='${dirname}' and trashed=false and '${rootdir}' in parents")" From 74ef5101bfe1fae5be691e6b4ea11003e2032aab Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Thu, 25 Jun 2020 08:36:34 +0530 Subject: [PATCH 5/6] upload.sh: Never show curl progress if -D/-q flag is used or stdout is not a tty --- upload.sh | 6 ++++-- utils.sh | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/upload.sh b/upload.sh index 2075bd8..3b8eb57 100755 --- a/upload.sh +++ b/upload.sh @@ -609,7 +609,7 @@ _setup_arguments() { _usage ;; -D | --debug) - DEBUG="true" + DEBUG="true" && CURL_ARGS="-s" export DEBUG ;; -u | --update) @@ -680,7 +680,7 @@ _setup_arguments() { fi ;; -q | --quiet) - QUIET="_print_center_quiet" && CURL_ARGS="-s" + QUIET="_print_center_quiet" ;; -v | --verbose) VERBOSE="true" @@ -743,6 +743,8 @@ _setup_arguments() { { [[ -n ${FOLDER_INPUT} && -z ${FOLDERNAME} ]] && FOLDERNAME="${FOLDER_INPUT}"; } || : { [[ -n ${VERBOSE_PROGRESS} && -n ${VERBOSE} ]] && unset "${VERBOSE}"; } || : + + { [[ -n ${QUIET} ]] && CURL_ARGS="-s"; } || : } ################################################### diff --git a/utils.sh b/utils.sh index 69dc9ff..de0c4fc 100755 --- a/utils.sh +++ b/utils.sh @@ -74,6 +74,7 @@ _check_debug() { trap 'shopt -s checkwinsize; (:;:)' SIGWINCH fi else + CURL_ARGS="-s" && export CURL_ARGS _print_center() { { [[ $# = 3 ]] && printf "%s\n" "[ ${2} ]"; } || { printf "%s\n" "[ ${2}${3} ]"; }; } _clear_line() { :; } fi From 9cc0d4ae62f0bea711f7105997162d5e8517b307 Mon Sep 17 00:00:00 2001 From: Akianonymus Date: Thu, 25 Jun 2020 09:49:26 +0530 Subject: [PATCH 6/6] gsync: Add systemd service configuration | Update readme with required steps --- README.md | 37 +++++++++++++++++++++++++++++++++++++ gsync.service | 15 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 gsync.service diff --git a/README.md b/README.md index e507377..c1bc1d4 100755 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ - [Synchronisation](#synchronisation) - [Basic Usage](#basic-usage) - [Sync Script Custom Flags](#sync-script-custom-flags) + - [Background Sync Job](#background-sync-jobs) - [Uninstall](#Uninstall) - [Reporting Issues](#reporting-issues) - [Contributing](#contributing) @@ -633,6 +634,42 @@ Note: Flags that use pid number as input should be used at last, if you are not +#### Background Sync Jobs + +There are basically two ways to start a background, first one we already covered in the above section. + +It will indefinitely run until and unless the host machine is rebooted. + +Now, a systemd service service can also be created which will start sync job after boot. + +1. Install the systemd unit file `sudo cp -v gsync.service /etc/systemd/system/` + +1. Open `/etc/systemd/system/gsync.service` with an editor of your choice and modify the following: + + - line 7, line 10 : User and ExecStart + + Change `user` to your username where gupload is installed + + - line 10 : ExecStart + + Change foldername to the local folder which will be synced + + Add additional flags like -d for gdrive_folder, etc as per the requirements, use whatever supported by gsync + + - line 11 : ExecStop + + Add everything which was added in line 10 ( ExecStart ) and add --kill to the end. + + e.g: If ExecStart was `ExecStart=/home/user/.google-drive-upload/bin/gsync /home/user/foldername -d drive_folder`, then ExecStop will be `ExecStart=/home/user/.google-drive-upload/bin/gsync /home/user/foldername -d drive_folder --kill` + +1. Reload the systemctl daemon so it can see your new systemd unit `sudo systemctl daemon-reload` + +1. Start the service `sudo systemctl start gsync` + + This is same as starting a sync job with command itself as mentioned in previous section. + +1. If you want the bot to automatically start on boot, run `sudo systemctl enable gsync` + ## Uninstall If you have followed the automatic method to install the script, then you can automatically uninstall the script. diff --git a/gsync.service b/gsync.service new file mode 100644 index 0000000..7c26ee7 --- /dev/null +++ b/gsync.service @@ -0,0 +1,15 @@ +[Unit] +Description=google-drive-upload synchronisation service +After=network.target + +[Service] +Type=simple +User=user +Restart=always +RestartSec=1 +ExecStart=/home/user/.google-drive-upload/bin/gsync /home/user/foldername -d drive_folder +ExecStop=/home/user/.google-drive-upload/bin/gsync /home/user/foldername -d drive_folder --kill + +[Install] +WantedBy=multi-user.target +