keystone-swift-curl

#!/bin/sh
# keystone-swift-curl (part of ossobv/vcutil) // wdoekes/2022-2024
#   // Public Domain
#
# Helper shell script to do simple GET/PUT/HEAD/DELETE in OpenStack Swift
# after performing Keystone authentication.
#
# Dependencies: curl (for http requests), jq (to extract project url)
#
# To use this, you make a configuration file:
#
#   SWIFT_KEYSTONE_URL=https://keystone.example.com/v3
#   SWIFT_AUTH_USER=username@domainname
#   SWIFT_AUTH_PASSWORD=password
#   SWIFT_PROJECT=projectname@domainname
#   #SWIFT_PROJECT_URL=https://swift.example.com/projecturl
#   SWIFT_CONTAINER=container
#
# Pass the configuration file as arg1, then method, then the remote file.
#
#   $ keystone-swift-curl path/to/config PUT remote/file/path \
#       -H 'Content-Type: application/octet-stream' < data
#   $ keystone-swift-curl path/to/config HEAD remote/file/path
#   $ keystone-swift-curl path/to/config GET remote/file/path > data
#   $ keystone-swift-curl path/to/config DELETE remote/file/path
#
# Or, alternatively, load this into your script:
#
#   SWIFT_KEYSTONE_URL=https://keystone.example.com/v3
#   # and the other variables..
#   . /usr/bin/keystone-swift-curl  # source this script as a library
#   swift_init                      # make sure you init
#   swift_login                     # log in / fetch token
#   if ! swift_head path/to/file 2>/dev/null; then
#       echo data | swift_put path/to/file \
#         -H 'Content-Type: application/octet-stream'
#   fi
#
# Put QUIET=1 in the environment to silence any up/down progress.
#
# Tips:
#
#   Get a file listing:
#   $ keystone-swift-curl path/to/config GET '' \
#       [-H 'Content-Type: application/json']
#
#   When using a filename from the listing, be sure to urlencode it when
#   using it in a new URL.
#   $ perl -e 'sub R{$n=ord($1);if($n<=32||$n==37||$n>=127){
#       sprintf("%%%02X",$n)}else{$1}};' -pe 's/(.)/R($1)/ge'
#
#   Getting CORS headers:
#   $ keystone-swift-curl path/to/config HEAD '' | grep -i x-container-meta
#       x-container-meta-access-control-allow-origin: https://...
#
#   Setting CORS headers:
#   $ keystone-swift-curl path/to/config POST '' \
#       -H 'X-Container-Meta-Access-Control-Allow-Origin:
#             https://domain1.example.com/ https://domain2.example.com/'
#
#   Removing CORS headers:
#   $ keystone-swift-curl path/to/config POST '' \
#       -H 'X-Remove-Container-Meta-Access-Control-Allow-Origin: x'
#
#   Other useful headers:
#   - X-Account-Meta-Temp-URL-Key (set SWIFT_CONTAINER=)
#   - X-Account-Meta-Temp-URL-Key-2 (set SWIFT_CONTAINER=)
#   - X-Container-Meta-Temp-URL-Key
#   - X-Container-Meta-Temp-URL-Key-2
#

swift_init() {
    # Check variables before starting. Works if 'set -u' is called.
    local var
    for var in $SWIFT_KEYSTONE_URL $SWIFT_AUTH_USER $SWIFT_AUTH_PASSWORD \
            $SWIFT_PROJECT $SWIFT_CONTAINER; do
        true
    done

    # NOTE: We use a temp directory to store credentials. This is safer
    # than passing these credentials on the command line, as they can be
    # seen in 'ps' output. (We do pass credentials in a printf call, but
    # because that is a shell builtin, no process is spawned for it.)
    if test -n "${SWIFT_RUNDIR:-}"; then
        echo "SWIFT_RUNDIR already set '($SWIFT_RUNDIR)'. Not implemented" >&2
        exit 1
    fi
    SWIFT_RUNDIR=$(mktemp -d)
    trap 'rm -rf "$SWIFT_RUNDIR"' EXIT
}

_swift_set_rundir() {
    # We allow multiple logins. Store the tokens in a subdirectory.
    # NOTE: If you want to use this functionality, you'll have to juggle
    # some globals.
    #
    # Example:
    #
    #     login_site_1() {
    #         SWIFT_PROJECT=site1@domain1
    #         SWIFT_PROJECT_URL=
    #         swift_login
    #         SWIFT_PROJECT_URL_SITE1=$SWIFT_PROJECT_URL
    #     }
    #     login_site_2() {
    #         SWIFT_PROJECT=site2@domain2
    #         SWIFT_PROJECT_URL=
    #         swift_login
    #         SWIFT_PROJECT_URL_SITE2=$SWIFT_PROJECT_URL
    #     }
    #     use_site1() {
    #         SWIFT_PROJECT=site1@domain1
    #         SWIFT_PROJECT_URL=$SWIFT_PROJECT_URL_SITE1
    #         SWIFT_CONTAINER=the_container
    #         swift_get 'file/from/site1'
    #     }
    #
    local hostname="${SWIFT_KEYSTONE_URL#*://}"
    hostname=${hostname%%/*}
    SWIFT_RUNDIR_PROJECT="$SWIFT_RUNDIR/$hostname/$SWIFT_PROJECT"
}

swift_login() {
    local user_domain user project_domain project password
    user=${SWIFT_AUTH_USER%@*}; user_domain=${SWIFT_AUTH_USER##*@}
    project=${SWIFT_PROJECT%@*}; project_domain=${SWIFT_PROJECT##*@}
    password=$SWIFT_AUTH_PASSWORD

    # Choose project scoped token. (For single-project accounts we could
    # skip the scope. For multi-project or sysadmin accounts, we can
    # access /auth/domains and /auth/projects but not /project and
    # (thus) not swift.)
    _swift_set_rundir
    mkdir -p "$SWIFT_RUNDIR_PROJECT"
    echo "\
    {\"auth\": {
      \"identity\": {
       \"methods\": [\"password\"],
       \"password\": {\"user\": {
         \"name\": \"$user\",
         \"domain\": {\"name\": \"$user_domain\"},
         \"project\": {\"name\": \"$project\", \"domain\": {
           \"name\": \"$project_domain\"}},
         \"password\": \"$password\"}}},
      \"scope\":{
       \"project\":{
        \"name\":\"$project\",
        \"domain\":{\"name\":\"$project_domain\"}}}
    }}" >"$SWIFT_RUNDIR_PROJECT/login"

    local output
    output=$(curl -fsS --max-time 4 --include \
        -H "Content-Type: application/json" \
        --data @"$SWIFT_RUNDIR_PROJECT/login" \
        "$SWIFT_KEYSTONE_URL/auth/tokens")

    printf '%s\n' "$output" |
        sed -ne 's/^x-subject-token:[[:blank:]]*/X-Auth-Token: /ip' \
          >"$SWIFT_RUNDIR_PROJECT/x-auth-token"
    if test -z "${SWIFT_PROJECT_URL:-}"; then
        output=$(printf '%s\n' "$output" | sed -e '1,/^[[:cntrl:]]*$/d')
        if command -v jq >/dev/null && SWIFT_PROJECT_URL=$(
                    _swift_get_project_url_from_tokens "$output"); then
            true
        else
            echo 'SWIFT_PROJECT_URL not set but jq(1) call failed/missing' >&2
            echo 'You can probably extract it from this manually:' >&2
            if command -v json_pp >/dev/null; then
                printf '%s\n' "$output" | json_pp >&2
            else
                printf '%s\n' "$output" >&2
            fi
            echo 'Look for the "url" of an "endpoint" in' >&2
            echo 'an "object-store" catalog' >&2
            exit 1
        fi
    fi
}

_swift_get_project_url_from_tokens() {
    # {"token": {
    #   "user": {...}
    #   "project": {"domain":{...},"id":"...","name":"..."}
    #   ...
    #   "catalog": [
    #    {"type":"identity","name":"keystone","endpoints":[{...}]},
    #    {"type":"object-store","name":"swift","endpoints":[{
    #      "id":"...","interface":"public","region_id":"NL1",
    #      "url": "https://...","region":"NL1"}]}],
    # }}
    local tokens_js="$1"
    local ret
    if ret=$(
        printf '%s\n' "$tokens_js" | jq -r '
            .token.catalog[] |
            select(.type=="object-store").endpoints[] |
            select(.interface=="public").url') && test -n "$ret"; then
        printf '%s\n' "$ret"
    else
        false
    fi
}

swift_do() {
    local project_url="$SWIFT_PROJECT_URL"
    local container="/$SWIFT_CONTAINER"
    container=${container%/}
    local meth="$1"  # GET|DELETE|HEAD|POST|PUT
    local file="$2"  # remote/path/to/file
    shift; shift
    _swift_set_rundir
    if test "$meth" = PUT && test "${QUIET:-0}" = 0; then
        # To get a progress bar, we must redirect stdout to /dev/null. cURL
        # feature..
        echo -n "$file " >&2
        curl --progress-bar -fX "$meth" \
            -H @"$SWIFT_RUNDIR_PROJECT/x-auth-token" \
            "$project_url$container/$file" "$@" >/dev/null
    else
        curl -fsSX "$meth" \
            -H @"$SWIFT_RUNDIR_PROJECT/x-auth-token" \
            "$project_url$container/$file" "$@"
    fi
}

swift_delete() { swift_do DELETE "$@"; }
swift_get() { swift_do GET "$@"; }
swift_head() {
    # BEWARE: The etag holds the md5sum of the file, except when the file is
    # re-assembled from a large object (DLO).
    echo "$1"
    swift_do HEAD "$@" -v -H 'Connection: close' 2>&1 |
        sed -e '/^</!d;s/</ /'
}
swift_post() { swift_do POST "$@"; }
swift_put() {
    if test -t 0; then
        echo "$0: Upload source is stdin. Did you forget to supply a file?" >&2
        exit 1
    fi

    local arg has_content_type=false
    for arg in "$@"; do
        test "${arg%%:*}" != 'Content-Type' || has_content_type=true
    done
    if ! $has_content_type; then
        echo "$0: Forgot -H 'Content-Type: application/octet-stream' ?" >&2
        echo "$0: Maybe use: file --mime-type FILE" >&2
    fi

    local file="$1"; shift
    local size='?'
    local divide

    if test -b /proc/self/fd/0; then
        # Block device; cannot get file size, is likely large.
        divide=true
    elif test -f /proc/self/fd/0; then
        # Real file? Get size?
        size=$(stat -Lc%s /proc/self/fd/0)
        if test $size -gt $((4096 * 1024 * 1024)); then
            # Larger than 4GB.
            divide=true
        else
            divide=false
        fi
    else
        # Not a real file. A stream? Cannot easily cut it up.
        divide=false
    fi

    if $divide; then
        # Dynamic Large Objects
        # https://docs.openstack.org/swift/latest/overview_large_objects.html
        local n=0
        local mb=$((1024 * 1024))
        local chunk_mbs=1024
        if test $size = '?'; then
            # Bah. This can be slow...
            echo "$(date -Iseconds): full read of $(
                readlink /proc/self/fd/0) for size..." >&2
            size=$(wc -c < /proc/self/fd/0)
            echo "$(date -Iseconds): size $size" >&2
        fi
        local chunk_last=$(( (size - 1) / $mb ))
        local chunks=$(seq 0 $chunk_mbs $chunk_last | wc -l)
        if test "${QUIET:-0}" = 0; then
            echo "$file -- uploading $chunks x 1GB chunks" >&2
        fi
        for chunk_off in $(seq 0 $chunk_mbs $chunk_last); do
            n=$((n + 1))
            # We could like to use --data-binary @- but that causes curl to run
            # out of memory on some systems. Using -T /dev/stdin does not. But
            # now curl does not "know" the filesize. The progress bar is now
            # not very informative.
            dd if=/proc/self/fd/0 bs=$mb skip=$chunk_off count=$chunk_mbs \
                status=none | swift_do PUT "$file/$(printf %08d $n)" \
                    -H "Transfer-Encoding: chunked" -T /dev/stdin "$@"
            if test "${QUIET:-0}" = 0; then
                printf '\033[F%s \n' "$(date -Iseconds)"  # on prev line
            fi
        done
        swift_do PUT "$file" -H "X-Object-Manifest: $SWIFT_CONTAINER/$file" \
            --data-binary ''
    else
        swift_do PUT "$file" -H "Transfer-Encoding: chunked" \
            -T /proc/self/fd/0 "$@"
        if test "${QUIET:-0}" = 0; then
            printf '\033[F%s \n' "$(date -Iseconds)"  # on prev line
        fi
    fi
}


if test "${0##*/}" = 'keystone-swift-curl'; then
    # We're the main script; we're not sourced by someone else.
    case ${1:---help} in
    -h|--help) echo "$0: usage: See $0 source" >&2; exit 1
    esac

    set -eu
    if test "${1#/}" = "$1"; then
        . "./$1"; shift # source the config file ($1, must contain slash)
    else
        . "$1"; shift   # source the config file ($1)
    fi
    swift_init          # initialize temp dir
    swift_login         # get token
    method=$1; shift    # get method ($2)
    case $method in
    DELETE) swift_delete "$@";;
    # TODO: create a LIST here where we multi-get using ?marker= to
    # quickly get a listing. (Both for json and text.)
    GET) swift_get "$@";;
    HEAD) swift_head "$@";;
    POST) swift_post "$@";;
    PUT) swift_put "$@";;
    *) echo "unknown method: $method" >&2; exit 1
    esac
fi