diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 0000000000..6cff1d4271 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,29 @@ +--- +name: 'Bug report' +about: 'Create a report to help us improve StreamPipes!' +labels: bug +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment (please complete the following information):** + - OS: [e.g. Ubuntu] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 0000000000..b383e1267a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,17 @@ +--- +name: 'Feature request' +about: 'Suggest a new idea for StreamPipes!' +labels: bug +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed1c5da43b..5af4fbc90e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ build: - echo "$GPG_PRIVATE_KEY" | gpg --batch --import --passphrase "$GPG_PASSPHRASE" - echo "$MAVEN_CREDENTIALS" > /root/.m2/settings.xml # - mvn clean package javadoc:aggregate gpg:sign -DskipTests - - mvn clean package javadoc:aggregate -DskipTests + - mvn clean package javadoc:aggregate - export MVN_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[') - "echo $MVN_VERSION >> ./target/mvn_version" artifacts: @@ -64,6 +64,7 @@ docker-backend: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $REGISTRY_HOST - docker login -u riemer -p $HARBOR_PASSWORD laus.fzi.de:8201 - docker build --pull -t $IMAGE_NAME/backend:latest -t $IMAGE_NAME/backend:$MVN_VERSION -t $HARBOR_IMAGE_NAME/backend:latest -t $HARBOR_IMAGE_NAME/backend:$MVN_VERSION ./streampipes-backend/ + - docker build --pull -t $IMAGE_NAME/backend:latest -t $IMAGE_NAME/backend:$MVN_VERSION ./streampipes-backend/ - docker push $IMAGE_NAME/backend:$MVN_VERSION - docker push $IMAGE_NAME/backend:latest - docker push $HARBOR_IMAGE_NAME/backend:$MVN_VERSION @@ -81,6 +82,7 @@ docker-connect-container: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $REGISTRY_HOST - docker login -u riemer -p $HARBOR_PASSWORD laus.fzi.de:8201 - docker build --pull -t $IMAGE_NAME/streampipes-connect-container:latest -t $IMAGE_NAME/streampipes-connect-container:$MVN_VERSION -t $HARBOR_IMAGE_NAME/streampipes-connect-container:latest -t $HARBOR_IMAGE_NAME/streampipes-connect-container:$MVN_VERSION ./streampipes-connect-container/ + - docker build --pull -t $IMAGE_NAME/streampipes-connect-container:latest -t $IMAGE_NAME/streampipes-connect-container:$MVN_VERSION ./streampipes-connect-container/ - docker push $IMAGE_NAME/streampipes-connect-container:$MVN_VERSION - docker push $IMAGE_NAME/streampipes-connect-container:latest - docker push $HARBOR_IMAGE_NAME/streampipes-connect-container:$MVN_VERSION diff --git a/.gitlab/issue_templates/bug.md b/.gitlab/issue_templates/bug.md new file mode 100644 index 0000000000..fb067dc9d5 --- /dev/null +++ b/.gitlab/issue_templates/bug.md @@ -0,0 +1,29 @@ +**Describe the bug** + +A clear and concise description of what the bug is. + +**To Reproduce** + +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**Screenshots** + +If applicable, add screenshots to help explain your problem. + +**Environment (please complete the following information):** + + - OS: [e.g. Ubuntu] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** + +Add any other context about the problem here. \ No newline at end of file diff --git a/.gitlab/issue_templates/feature.md b/.gitlab/issue_templates/feature.md new file mode 100644 index 0000000000..d5ac51c912 --- /dev/null +++ b/.gitlab/issue_templates/feature.md @@ -0,0 +1,15 @@ +**Is your feature request related to a problem? Please describe.** + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** + +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** + +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** + +Add any other context or screenshots about the feature request here. diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..734d804aaf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: java + +addons: + sonarcloud: + organization: "streampipes" + token: + secure: $SONAR_TOKEN + branches: + - dev + +script: + - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar --quiet + diff --git a/README.md b/README.md index 4d9e75ec67..81ab072901 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Travis Badge](https://travis-ci.org/streampipes/streampipes-ce.svg?branch=dev)](https://travis-ci.org/streampipes/streampipes-ce.svg?branch=dev) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/34a7e26be4fc4fa284ee5201b6d386ea)](https://www.codacy.com/app/dominikriemer/streampipes-ce?utm_source=github.com&utm_medium=referral&utm_content=streampipes/streampipes-ce&utm_campaign=Badge_Grade) [![Docker pulls](https://img.shields.io/docker/pulls/streampipes/backend.svg)](https://hub.docker.com/r/streampipes/backend/) [![Maven central](https://img.shields.io/maven-central/v/org.streampipes/streampipes-backend.svg)](https://img.shields.io/maven-central/v/org.streampipes/streampipes-backend.svg) diff --git a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml index 1d65adb5ed..96e85351a3 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 ../../pom.xml streampipes-archetype-pe-processors-flink diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml index 6d3c28f28e..0ce2d43fa8 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.61.0 + 0.62.0 diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml index cc27d78836..bdbd6e3654 100644 --- a/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 ../../pom.xml streampipes-archetype-pe-processors-jvm diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml index e48cdacf3f..fca1621f42 100644 --- a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.61.0 + 0.62.0 diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml index dbc1575ea3..309be691a7 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 ../../pom.xml streampipes-archetype-pe-sinks-flink diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml index 7666a391cf..01eea214cf 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.61.0 + 0.62.0 diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml index 08aa3325c9..28ae8cf047 100644 --- a/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 ../../pom.xml streampipes-archetype-pe-sinks-jvm diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml index e48cdacf3f..fca1621f42 100644 --- a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.61.0 + 0.62.0 diff --git a/archetypes/streampipes-archetype-pe-sources/pom.xml b/archetypes/streampipes-archetype-pe-sources/pom.xml index 4519c99647..6e3a5d8a8b 100644 --- a/archetypes/streampipes-archetype-pe-sources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sources/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 ../../pom.xml streampipes-archetype-pe-sources diff --git a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml index bca7da37c2..d137a8c0f0 100644 --- a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.61.0 + 0.62.0 diff --git a/install.sh b/install.sh new file mode 100755 index 0000000000..266955956c --- /dev/null +++ b/install.sh @@ -0,0 +1,420 @@ +#!/bin/sh +set -e + +# Usage: +# curl ... | ENV_VAR=... sh - +# or +# ENV_VAR=... ./install.sh +# +# Example: +# Directory to installing StreamPipes binary (needs sudo) +# curl ... | INSTALL_SP_BIN_DIR="/usr/local/bin" sh - +# +# Environment variables: +# +# - INSTALL_SP_BIN_DIR +# Directory to install StreamPipes binary +# default: /usr/local/bin + +GIT_CLI_URL=https://github.com/streampipes/streampipes-cli/tarball/master +#TODO: change SP_BACKEND_VERSION based on Maven Version in gitlab-ci.yml +#SP_BACKEND_VERSION=0.60.0 +DEBUG=false + +if [ "$1" = "--debug" ]; then + DEBUG=true +fi + +# --- helper functions for logs --- +info(){ + echo "[INFO]\t" "$@" +} + +debug() { + if $DEBUG; then + echo "[DEBUG]\t" "$@" + fi +} + +warning() { + echo "[WARN]\t" "$@" +} + +fatal(){ + echo "[ERROR]\t" "$@" + exit 1 +} + +install_notice() { + echo + echo + echo " StreamPipes CE will now be installed on your system" + echo + echo +} + +uninstall_notice() { + echo + echo + echo " To uninstall StreamPipes CE run the following command:" + echo + echo " Linux:" + echo " \$ sudo sp-uninstall " + echo + echo " Mac:" + echo " \$ sp-uninstall " + echo +} + +# --- helper functions --- +semverParseDocker() { + major_docker="${1%%.*}" + minor_docker="${1#$major_docker.}" + minor_docker="${minor_docker%%.*}" + patch_docker="${1#$major_docker.$minor_docker.}" + patch_docker="${patch_docker%%[-.]*}" +} + +semverParseDockerCompose() { + major_docker_compose="${1%%.*}" + minor_docker_compose="${1#$major_docker_compose.}" + minor_docker_compose="${minor_docker_compose%%.*}" +} + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +check_and_add_to_path() { + SP_HOME=$1 + debug "Add SP_HOME to PATH" + case ":${PATH:=$SP_HOME}:" in + *:$SP_HOME:*) + debug "SP_HOME found in PATH" + ;; + *) + s=$(echo $SHELL) + currShell=${s##*/} + case $currShell in + "bash") + debug "Detected shell: $currShell" + if [ $2 = "user-only" ] ; then + debug "Check if SP_HOME exists in $HOME/.bashrc" + + if grep -q SP_HOME "$HOME/.bashrc"; then + # found + info "[SKIPPED] SP_HOME already set" + else + # not found + info "Add SP_HOME to $HOME/.bashrc" + echo "export SP_HOME=$SP_HOME" >> $HOME/.bashrc + echo 'export PATH=$PATH:$SP_HOME' >> $HOME/.bashrc + fi + + elif [ $2 = "system-wide" ]; then + debug "Check if SP_HOME exists in /etc/profile.d/streampipes-env.sh" + + if [ -f "/etc/profile.d/streampipes-env.sh" ]; then + # found + info "[SKIPPED] SP_HOME already set" + else + # not found + info "Add SP_HOME to /etc/profile.d/streampipes-env.sh" + tee /etc/profile.d/streampipes-env.sh >/dev/null << EOF +#!/bin/sh +SP_HOME="$SP_HOME" +if [ -d "\$SP_HOME" ] ; then + PATH="\$SP_HOME:\$PATH" +fi +EOF + chmod 755 /etc/profile.d/streampipes-env.sh + fi + + else + warning "SP_HOME not set for $currShell. Set manually" + fi + ;; + "zsh") + debug "Detected shell: $currShell" + if [ $2 == "user-only" ]; then + debug "Check if SP_HOME exists in $HOME/.zshrc" + + if grep -q SP_HOME "$HOME/.zshrc"; then + # found + info "[SKIPPED] SP_HOME already set" + else + # not found + info "Add SP_HOME to $HOME/.zshrc" + echo "export SP_HOME=$SP_HOME" >> $HOME/.zshrc + echo 'export PATH=$PATH:$SP_HOME' >> $HOME/.zshrc + fi + + elif [ $2 == "system-wide" ]; then + debug "Check if SP_HOME exists in /etc/zsh/zshenv" + + if grep -q SP_HOME "/etc/zsh/zshenv"; then + # found + info "[SKIPPED] SP_HOME already set" + else + # not found + info "Add SP_HOME to /etc/zsh/zshenv" + echo "export SP_HOME=$SP_HOME" >> /etc/zsh/zshenv + echo 'export PATH=$PATH:$SP_HOME' >> /etc/zsh/zshenv + fi + + else + warning "SP_HOME not set for $currShell. Set manually" + fi + ;; + *) + warning "Could not detect shell environment. Manually export SP_HOME=$SP_HOME and add to PATH" + ;; + esac + ;; + esac +} + +# --- functions --- +setup_env() { + + if [ $OS_TYPE = "Linux" ]; then + SP_HOME="/opt/streampipes" + if [ ! -d $SP_HOME ]; then + info "Create and set StreamPipes Home (SP_HOME): $SP_HOME" + $SUDO mkdir -p $SP_HOME + + check_and_add_to_path $SP_HOME system-wide + + else + info "[SKIPPED] StreamPipes Home already exists" + check_and_add_to_path $SP_HOME system-wide + + fi + elif [ $OS_TYPE = "Mac" ]; then + SP_HOME="$HOME/streampipes" + if [ ! -d $SP_HOME ]; then + info "Create and set StreamPipes Home (SP_HOME): $SP_HOME" + mkdir -p $SP_HOME + + check_and_add_to_path $SP_HOME user-only + + else + info "[SKIPPED] StreamPipes Home already exists" + check_and_add_to_path $SP_HOME user-only + + fi + fi + + # --- use binary install directory if defined or create default --- + if [ -n "${INSTALL_SP_BIN_DIR}" ]; then + BIN_DIR="${INSTALL_SP_BIN_DIR}" + else + BIN_DIR="/usr/local/bin" + fi + + UNINSTALL_SP_SH=sp-uninstall + + # --- use sudo if we are not already root --- + SUDO=sudo + if [ `id -u` = 0 ]; then + SUDO= + fi + +} + +# --- fatal if no curl --- +verify_curl() { + info "Verifying curl" + if [ -z `which curl || true` ]; then + fatal "Cannot find curl for downloading files" + fi +} + +# --- fatal if architecture not supported --- +verify_arch() { + info "Verifying system architecture" + ARCH=`uname -m` + case $ARCH in + amd64) + ARCH=amd64 + SUFFIX= + debug "Supported architecture detected: $ARCH" + ;; + x86_64) + ARCH=amd64 + SUFFIX= + debug "Supported architecture detected: $ARCH" + ;; + *) + fatal "Unsupported architecture: $ARCH" + esac +} + +# --- fatal if OS not supported --- +verify_os() { + info "Verifying OS" + OS_TYPE="$(uname -s)" + case $OS_TYPE in + Linux*) + OS_TYPE=Linux + debug "Supported OS detected: $OS_TYPE" + ;; + Darwin*) + OS_TYPE=Mac + debug "Supported OS detected: $OS_TYPE" + ;; + *) + fatal "Unsupported O: $OS_TYPE" + esac +} + +# --- fatal if Docker/Docker Compose not installed or version mismatch --- +verify_docker() { + info "Verifying Docker and Docker Compose" + if command_exists docker && command_exists docker-compose; then + docker_version=`docker -v | cut -d ' ' -f3 | cut -d ',' -f1` + docker_compose_version=`docker-compose -v | cut -d ' ' -f3 | cut -d ',' -f1` + + MAJOR_W_DOCKER=1 + MINOR_W_DOCKER=10 + + MAJOR_W_DOCKER_COMPOSE=1 + MINOR_W_DOCKER_COMPOSE=8 + + semverParseDocker "$docker_version" + semverParseDockerCompose "$docker_compose_version" + + shouldWarnDocker=0 + if [ "$major_docker" -lt "$MAJOR_W_DOCKER" ]; then + shouldWarnDocker=1 + fi + + if [ "$major_docker" -le "$MAJOR_W_DOCKER" ] && [ "$minor_docker" -lt "$MINOR_W_DOCKER" ]; then + shouldWarnDocker=1 + fi + + shouldWarnDockerCompose=0 + if [ "$major_docker_compose" -lt "$MAJOR_W_DOCKER_COMPOSE" ]; then + shouldWarnDockerCompose=1 + fi + + if [ "$major_docker_compose" -le "$MAJOR_W_DOCKER_COMPOSE" ] && [ "$minor_docker_compose" -lt "$MINOR_W_DOCKER_COMPOSE" ]; then + shouldWarnDockerCompose=1 + fi + + if [ $shouldWarnDocker -eq 1 ]; then + fatal "Docker version $docker_version detected which is not compatible. Supported Docker version from $MAJOR_W_DOCKER.$MINOR_W_DOCKER.0+" + fi + + if [ $shouldWarnDockerCompose -eq 1 ]; then + fatal "Docker Compose version $docker_compose_version detected which is not compatible. Supported Docker Compose version from $MAJOR_W_DOCKER_COMPOSE.$MINOR_W_DOCKER_COMPOSE.0+" + fi + + debug "Installed Docker version: $docker_version" + debug "Installed Docker Compose version: $docker_compose_version" + else + fatal "Cannot find Docker and/or Docker Compose. Please make sure Docker and Docker Compose are installed and configured properly" + fi +} + +download_and_configure() { + CLI_DIR=$SP_HOME/streampipes-cli + if [ ! -d $CLI_DIR ]; then + info "Create directory for StreamPipes CLI in SP_HOME" + mkdir $CLI_DIR + fi + + if [ ! "$(ls -A $CLI_DIR)" ]; then + # SP_HOME empty + info "Downloading StreamPipes CLI to SP_HOME" + curl -sSfL ${GIT_CLI_URL} | tar -xzf - -C $CLI_DIR --strip-components=1 || fatal "Error while downloading StreamPipes CLI project" + info "Copy StreamPipes CLI binary to ${BIN_DIR}/sp" + cp $CLI_DIR/sp $BIN_DIR + else + info "[SKIPPED] StreamPipes CLI already exists" + fi +} + +# --- create uninstall script --- +create_uninstall() { + info "Creating StreamPipes uninstall script in ${BIN_DIR}/${UNINSTALL_SP_SH}" + tee ${BIN_DIR}/${UNINSTALL_SP_SH} >/dev/null << EOF +#!/bin/sh + +# --- helper functions for logs --- +info(){ + echo "[INFO]\t" "\$@" +} + +fatal(){ + echo "[ERROR]\t" "\$@" + exit 1 +} + +s=$(echo \$SHELL) +currShell=\${s##*/} + +case \$currShell in + "zsh") + if [ -f \$HOME/.zshrc ]; then + info "Removing SP_HOME from \$HOME/.zshrc" + sed -i.bak '/SP_HOME/d' \$HOME/.zshrc + rm \$HOME/.zshrc.bak + elif [ -f /etc/zsh/zshenv ]; then + info "Removing SP_HOME from /etc/zsh/zshenv" + sed -i.bak '/SP_HOME/d' /etc/zsh/zshenv + rm /etc/zsh/zshenv.bak + fi + ;; + "bash") + if [ -f \$HOME/.bashrc ]; then + info "Removing SP_HOME from \$HOME/.bashrc" + sed -i.bak '/SP_HOME/d' \$HOME/.bashrc + rm \$HOME/.bashrc + elif [ -f /etc/profile.d/streampipes-env.sh ]; then + info "Deleting /etc/profile.d/streampipes-env.sh" + rm /etc/profile.d/streampipes-env.sh + fi + ;; + *) + fatal "Could not unset SP_HOME from \$currShell" +esac + +info "Deleting StreamPipes Home directory ${SP_HOME}" +rm -rf ${SP_HOME} +info "Deleting StreamPipes CLI ${BIN_DIR}/sp" +rm -f ${BIN_DIR}/sp +info "Deleting StreamPipes uninstall script ${BIN_DIR}/${UNINSTALL_SP_SH}" +rm -rf ${BIN_DIR}/${UNINSTALL_SP_SH} +EOF + #$SUDO chmod 755 ${SP_HOME}/${UNINSTALL_SP_SH} + #$SUDO chown root:root ${SP_HOME}/${UNINSTALL_SP_SH} + chmod 755 ${BIN_DIR}/${UNINSTALL_SP_SH} + chown $USER:$USER ${BIN_DIR}/${UNINSTALL_SP_SH} +} + +# --- start StreamPipes CLI script --- +start_cli() { + info "Starting StreamPipes CLI" + if command_exists sp; then + sp start + else + fatal "Could not find StreamPipes CLI binary" + fi +} + +# --- run install process --- +do_install () { + install_notice + verify_curl + verify_arch + verify_os + verify_docker + setup_env + download_and_configure + create_uninstall + uninstall_notice + start_cli +} + +do_install diff --git a/pom.xml b/pom.xml index ea990864ba..cc238b2eaa 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 pom UTF-8 diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000000..15517e5601 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,7 @@ +sonar.projectKey=streampipes_streampipes-ce +sonar.projectName=StreamPipes +sonar.projectVersion=0.61.1-SNAPSHOT + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +# This property is optional if sonar.modules is set. +sonar.sources=. \ No newline at end of file diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml index bbf963ffbc..a3ba1e9160 100644 --- a/streampipes-app-file-export/pom.xml +++ b/streampipes-app-file-export/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 StreamPipes App File Export streampipes-app-file-export @@ -66,6 +66,11 @@ 2.3.2 test + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.2 + \ No newline at end of file diff --git a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchConfig.java b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchConfig.java index d22718bbbd..a8827c1d12 100644 --- a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchConfig.java +++ b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchConfig.java @@ -32,7 +32,7 @@ public enum ElasticsearchConfig { config = SpConfig.getSpConfig("storage/elasticsearch"); config.register(HOST, "elasticsearch", "Hostname for the elasticsearch service"); - config.register(PORT, "9200", "Port for the elasticsearch service"); + config.register(PORT, 9200, "Port for the elasticsearch service"); config.register(PROTOCOL, "http", "Protocol the elasticsearch service"); config.register(DATA_LOCATION,"/home/user/", "Folder that stores all the created data blobs"); } @@ -41,8 +41,8 @@ public String getElasticsearchHost() { return config.getString(HOST); } - public String getElasticsearchPort() { - return config.getString(PORT); + public Integer getElasticsearchPort() { + return config.getInteger(PORT); } public String getElasticsearchURL() { diff --git a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/converter/JsonConverter.java b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/converter/JsonConverter.java index bf5f40052e..429ad10afc 100644 --- a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/converter/JsonConverter.java +++ b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/converter/JsonConverter.java @@ -28,42 +28,38 @@ public class JsonConverter { private JsonObject elasticJsonRepresentation; + private JsonParser jsonParser; - public JsonConverter(String elasticJsonRepresentation) { - this.elasticJsonRepresentation = new JsonParser().parse(elasticJsonRepresentation).getAsJsonObject(); + public JsonConverter() { + this.jsonParser = new JsonParser(); } - public String convertToJson() { - return extractContent().toString(); - } + public String getCsvHeader(String elasticJsonRepresentation) { + JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); + + Set> elements = inContent.entrySet(); + StringJoiner sj = new StringJoiner(";"); - public String convertToCsv() { - JsonArray inContent = extractContent(); - StringBuilder sb = new StringBuilder(); - - for(int i = 0; i < inContent.size(); i++) { - JsonObject jsonObject = inContent.get(i).getAsJsonObject(); - Set> elements = jsonObject.entrySet(); - StringJoiner sj = new StringJoiner(";"); - for (Map.Entry entry: elements) { - sj.add(entry.getValue().toString()); - } - sb.append(sj.toString()); - sb.append("\n"); + for (Map.Entry entry: elements) { + sj.add(entry.getKey().toString()); } - return sb.toString(); + return sj.toString() + "\n"; + } - private JsonArray extractContent() { - JsonArray inContent = elasticJsonRepresentation.get("hits").getAsJsonObject().get("hits").getAsJsonArray(); - JsonArray outContent = new JsonArray(); + public String convertToCsv(String elasticJsonRepresentation) { + JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); - for(int i = 0; i < inContent.size(); i++) { - JsonObject jsonObject = inContent.get(i).getAsJsonObject().get("_source").getAsJsonObject(); - outContent.add(jsonObject); + Set> elements = inContent.entrySet(); + StringJoiner sj = new StringJoiner(";"); + + for (Map.Entry entry: elements) { + sj.add(entry.getValue().toString()); } - return outContent; + return sj.toString() + "\n"; + } + } diff --git a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java index a163fe3ece..817975061a 100644 --- a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java +++ b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java @@ -25,7 +25,18 @@ import com.mashape.unirest.http.JsonNode; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; -import com.mashape.unirest.request.HttpRequestWithBody; +import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; +import org.elasticsearch.action.search.*; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.lightcouch.CouchDbClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,9 +47,7 @@ import org.streampipes.app.file.export.model.IndexInfo; import org.streampipes.storage.couchdb.utils.Utils; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; +import java.io.*; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -76,40 +85,35 @@ public Response createFiles(ElasticsearchAppData data) { boolean allData = data.isAllData(); try { - String countUrl = ElasticsearchConfig.INSTANCE.getElasticsearchURL() + "/" + index + "/_count"; - HttpResponse jsonResponse = unirestGet(countUrl); - String count = jsonResponse.getBody().getObject().get("count").toString(); - - String url = ElasticsearchConfig.INSTANCE.getElasticsearchURL() + "/" + index + "/_search"; - String response; - HttpRequestWithBody request = Unirest.post(url) - .header("accept", "application/json") - .header("Content-Type", "application/json"); - - if (allData) { - jsonResponse = request.body("{\"from\" : 0, \"size\" :" + count + ", \"query\": { \"match_all\": {} }}") - .asJson(); - timestampFrom = 0; - timeStampTo = 0; - } else { - jsonResponse = request.body("{\"from\" : 0, \"size\" :" + count + ", \"query\": {\"range\" : {\"timestamp\" : {\"gte\" : " + timestampFrom + ",\"lte\" : " + timeStampTo + "}}}}") - .asJson(); - } + RestHighLevelClient client = getRestHighLevelClient(); - response = jsonResponse.getBody().getObject().toString(); + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - if (("csv").equals(output)) { - response = new JsonConverter(response).convertToCsv(); - } else { - response = new JsonConverter(response).convertToJson(); + if (!allData) { + searchSourceBuilder.query(QueryBuilders.rangeQuery("timestamp").from(timestampFrom).to(timeStampTo)); } + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + //Time created in milli sec, index, from, to long timestamp = System.currentTimeMillis(); - String fileName = System.currentTimeMillis() + "-" + index + "-" + timestampFrom + "-" + timeStampTo + "." +output; + String fileName = System.currentTimeMillis() + "-" + index + "-" + timestampFrom + "-" + timeStampTo + "." + output; String filePath = mainFilePath + fileName; + FileOutputStream fileStream = this.getFileStream(filePath); - this.saveFile(filePath, response); + if(("csv").equals(output)) { + processCSV(client, fileStream, scrollId, scroll, searchHits); + } else { + processJSON(client, fileStream, scrollId, scroll, searchHits); + } + + fileStream.close(); CouchDbClient couchDbClient = getCouchDbClient(); Map map = new HashMap<>(); @@ -125,7 +129,7 @@ public Response createFiles(ElasticsearchAppData data) { return Response.ok().build(); - } catch (IOException | UnirestException e) { + } catch (IOException e) { e.printStackTrace(); LOG.error(e.getMessage()); return Response.status(500).entity(e).build(); @@ -206,13 +210,11 @@ private CouchDbClient getCouchDbClient() { return Utils.getCouchDbElasticsearchFilesEndppointClient(); } - private void saveFile(String filePath, String text) throws IOException { + private FileOutputStream getFileStream(String filePath) throws IOException { File file = new File(filePath); file.getParentFile().mkdirs(); FileWriter fileWriter = new FileWriter(file, true); - fileWriter.write(text); - fileWriter.flush(); - fileWriter.close(); + return new FileOutputStream(filePath); } private HttpResponse unirestGet(String url) throws UnirestException { @@ -223,4 +225,91 @@ private HttpResponse unirestGet(String url) throws UnirestException { return jsonResponse; } -} + private RestHighLevelClient getRestHighLevelClient() { + String host = ElasticsearchConfig.INSTANCE.getElasticsearchHost(); + int port = ElasticsearchConfig.INSTANCE.getElasticsearchPort(); + + return new RestHighLevelClient( + RestClient.builder( + new HttpHost(host, port, "http")) + .setRequestConfigCallback( + new RestClientBuilder.RequestConfigCallback() { + @Override + public RequestConfig.Builder customizeRequestConfig( + RequestConfig.Builder requestConfigBuilder) { + return requestConfigBuilder + .setConnectTimeout(5000) + .setSocketTimeout(60000); + } + }) + ); + + } + + private void processJSON(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, SearchHit[] searchHits) throws IOException { + fileStream.write("[".getBytes()); + boolean isFirstElement = true; + for (SearchHit hit : searchHits) { + if(!isFirstElement) + fileStream.write(",".getBytes()); + fileStream.write(hit.getSourceAsString().getBytes()); + isFirstElement = false; + } + + while (searchHits != null && searchHits.length > 0) { + + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + for (SearchHit hit : searchHits) { + fileStream.write(",".getBytes()); + fileStream.write(hit.getSourceAsString().getBytes()); + } + } + fileStream.write("]".getBytes()); + + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + + + } + + private void processCSV(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, + SearchHit[] searchHits) throws IOException { + JsonConverter jsonConverter = new JsonConverter(); + + boolean isFirstElement = true; + for (SearchHit hit : searchHits) { + if (isFirstElement) + fileStream.write(jsonConverter.getCsvHeader(hit.getSourceAsString()).getBytes()); + String response = jsonConverter.convertToCsv(hit.getSourceAsString()); + fileStream.write(response.getBytes()); + isFirstElement = false; + + } + + while (searchHits != null && searchHits.length > 0) { + + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + for (SearchHit hit : searchHits) { + fileStream.write(jsonConverter.convertToCsv(hit.getSourceAsString()).getBytes()); + } + + } + + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + + } + + + + } diff --git a/streampipes-backend/development/.env b/streampipes-backend/development/.env index 4a18cbeca6..506eb026d3 100644 --- a/streampipes-backend/development/.env +++ b/streampipes-backend/development/.env @@ -4,6 +4,9 @@ SP_KAFKA_HOST=localhost SP_ZOOKEEPER_HOST=localhost SP_JMS_HOST=localhost SP_KAFKA_REST_HOST=localhost +SP_KAFKA_REST_PORT=8073 SP_BACKEND_HOST=localhost SP_ELASTICSEARCH_HOST=localhost -SP_ASSETS_DIR=./assets \ No newline at end of file +SP_ASSETS_DIR=./assets +SP_DATALAKE_HOST=localhost +SP_DATALAKE_PORT=9200 diff --git a/streampipes-backend/pom.xml b/streampipes-backend/pom.xml index d8698e8e89..6593487e12 100644 --- a/streampipes-backend/pom.xml +++ b/streampipes-backend/pom.xml @@ -3,7 +3,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-backend war diff --git a/streampipes-code-generation/pom.xml b/streampipes-code-generation/pom.xml index 719c43e74c..c0f8f9d8bd 100644 --- a/streampipes-code-generation/pom.xml +++ b/streampipes-code-generation/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-code-generation diff --git a/streampipes-commons/pom.xml b/streampipes-commons/pom.xml index fe96ce663f..a5f031bc5c 100644 --- a/streampipes-commons/pom.xml +++ b/streampipes-commons/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-commons StreamPipes Commons diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileGenerator.java b/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileGenerator.java index 5007371a94..eb74d0c3f9 100644 --- a/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileGenerator.java +++ b/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileGenerator.java @@ -39,7 +39,6 @@ public class ZipFileGenerator { public ZipFileGenerator(File inputDirectory, File outputFile) { this(inputDirectory); this.outputFile = outputFile; - } public ZipFileGenerator(File inputDirectory) { diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml index decfde3bd1..ebde2014c5 100644 --- a/streampipes-config/pom.xml +++ b/streampipes-config/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java index e5b3197f14..908eed0195 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java @@ -49,7 +49,12 @@ public enum BackendConfig { config.register(BackendConfigKeys.KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); config.register(BackendConfigKeys.ASSETS_DIR, "/streampipes-assets", "The directory where " + "pipeline element assets are stored."); + config.register(BackendConfigKeys.DATA_LAKE_HOST, "elasticsearch", "The host of the data base used for the data lake"); + config.register(BackendConfigKeys.DATA_LAKE_PORT, 9200, "The port of the data base used for the data lake"); + config.register(BackendConfigKeys.INFLUX_HOST, "influxdb", "The host of the influx data base"); + config.register(BackendConfigKeys.INFLUX_PORT, 8086, "The hist of the influx data base"); + config.register(BackendConfigKeys.INFLUX_DATA_BASE, "sp", "The influx data base name"); } public String getBackendHost() { @@ -140,8 +145,33 @@ public String getAssetDir() { return config.getString(BackendConfigKeys.ASSETS_DIR); } + public String getDatalakeHost() { + return config.getString(BackendConfigKeys.DATA_LAKE_HOST); + } + + public int getDatalakePort() { + return config.getInteger(BackendConfigKeys.DATA_LAKE_PORT); + } + public String getDataLakeUrl() { + return getDatalakeHost() + ":" + getDatalakePort(); + } + + public String getInfluxHost() { + return config.getString(BackendConfigKeys.INFLUX_HOST); + } + public int getInfluxPort() { + return config.getInteger(BackendConfigKeys.INFLUX_PORT); + } + + public String getInfluxUrl() { + return "http://" + getInfluxHost() + ":" + getInfluxPort(); + } + + public String getInfluxDatabaseName() { + return config.getString(BackendConfigKeys.INFLUX_DATA_BASE); + } } diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java index 54080b342c..292b5203fd 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java @@ -33,6 +33,13 @@ public class BackendConfigKeys { public static final String KAFKA_REST_HOST = "SP_KAFKA_REST_HOST"; public static final String KAFKA_REST_PORT = "SP_KAFKA_REST_PORT"; public static final String ASSETS_DIR = "SP_ASSETS_DIR"; + public static final String DATA_LAKE_HOST = "SP_DATA_LAKE_HOST"; + public static final String DATA_LAKE_PORT = "SP_DATA_LAKE_PORT"; + + public static final String INFLUX_PORT = "SP_INFLUX_PORT"; + public static final String INFLUX_HOST = "SP_INFLUX_HOST"; + public static final String INFLUX_DATA_BASE = "SP_INFLUX_DATA_BASE"; + public static final String SERVICE_NAME = "SP_SERVICE_NAME"; } diff --git a/streampipes-connect-container/pom.xml b/streampipes-connect-container/pom.xml index 6676f6c3c2..64b3b10878 100644 --- a/streampipes-connect-container/pom.xml +++ b/streampipes-connect-container/pom.xml @@ -3,7 +3,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java index 62d3ec06c3..7c36e3d438 100644 --- a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java @@ -40,6 +40,9 @@ public GuessSchema guessSchema(AdapterDescription adapterDescription) throws Ada GuessSchema guessSchema; try { guessSchema = adapter.getSchema(adapterDescription); + for (int i = 0; i < guessSchema.getEventSchema().getEventProperties().size(); i++) { + guessSchema.getEventSchema().getEventProperties().get(i).setIndex(i); + } } catch (ParseException e) { logger.error(e.toString()); diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java index 51160e8ef0..0307238d1f 100644 --- a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java @@ -24,12 +24,10 @@ import org.streampipes.connect.adapter.generic.format.xml.XmlParser; import org.streampipes.connect.adapter.generic.protocol.set.HttpProtocol; import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; -import org.streampipes.connect.adapter.specific.sensemap.OpenSenseMapAdapter; import org.streampipes.connect.exception.AdapterException; import org.streampipes.model.connect.adapter.AdapterDescription; import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; -import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; import org.streampipes.model.connect.grounding.FormatDescription; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.rules.value.UnitTransformRuleDescription; @@ -63,16 +61,16 @@ public void getGenericAdapterSetDescription() throws AdapterException { assertEquals(GenericAdapterSetDescription.ID, a.getUri()); } - @Test - public void getSpecificAdapterStreamDescription() throws AdapterException { - AdapterDescription specificAdapterStreamDescription = new OpenSenseMapAdapter().declareModel(); - String jsonLd = JsonLdUtils.toJsonLD(specificAdapterStreamDescription); - - AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); - - assertTrue(a instanceof SpecificAdapterStreamDescription); - assertEquals(OpenSenseMapAdapter.ID, a.getUri()); - } +// @Test +// public void getSpecificAdapterStreamDescription() throws AdapterException { +// AdapterDescription specificAdapterStreamDescription = new OpenSenseMapAdapter().declareModel(); +// String jsonLd = JsonLdUtils.toJsonLD(specificAdapterStreamDescription); +// +// AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); +// +// assertTrue(a instanceof SpecificAdapterStreamDescription); +// assertEquals(OpenSenseMapAdapter.ID, a.getUri()); +// } @Test public void getFormatDescriptionHttpProtocolXmlFormat() throws AdapterException { diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java index 7532354add..0e0898223d 100644 --- a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java @@ -17,9 +17,6 @@ package org.streampipes.connect.management; -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.any; - import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -33,21 +30,21 @@ import org.streampipes.connect.exception.AdapterException; import org.streampipes.connect.management.worker.AdapterWorkerManagement; import org.streampipes.connect.utils.Utils; -import org.streampipes.model.connect.adapter.AdapterDescription; -import org.streampipes.model.connect.adapter.AdapterSetDescription; -import org.streampipes.model.connect.adapter.AdapterStreamDescription; -import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; -import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; -import org.streampipes.model.connect.adapter.SpecificAdapterSetDescription; +import org.streampipes.model.connect.adapter.*; import org.streampipes.model.connect.guess.GuessSchema; +import java.util.ArrayList; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; + @RunWith(PowerMockRunner.class) @PrepareForTest({ AdapterRegistry.class }) public class AdapterWorkerManagementTest { @Test public void startStreamAdapterSucess() throws AdapterException { - TestAdapter testAdapter = new TestAdapter(); + TestAdapter testAdapter = getTestAdapterInstance(); PowerMockito.mockStatic(AdapterRegistry.class); Mockito.when(AdapterRegistry.getAdapter(any(AdapterDescription.class))) @@ -78,7 +75,7 @@ public void stopStreamAdapterFail() { @Test public void stopStreamAdapterSuccess() throws AdapterException { - TestAdapter testAdapter = new TestAdapter(); + TestAdapter testAdapter = getTestAdapterInstance(); RunningAdapterInstances.INSTANCE.addAdapter("http://t.de/", testAdapter); AdapterWorkerManagement adapterWorkerManagement = new AdapterWorkerManagement(); adapterWorkerManagement.stopStreamAdapter(Utils.getMinimalStreamAdapter()); @@ -105,7 +102,8 @@ public void stopSetAdapterFail() { @Test public void stopSetAdapterSuccess() throws AdapterException { - TestAdapter testAdapter = new TestAdapter(); + TestAdapter testAdapter = getTestAdapterInstance(); + RunningAdapterInstances.INSTANCE.addAdapter("http://t.de/", testAdapter); AdapterWorkerManagement adapterWorkerManagement = new AdapterWorkerManagement(); adapterWorkerManagement.stopSetAdapter(Utils.getMinimalSetAdapter()); @@ -113,10 +111,19 @@ public void stopSetAdapterSuccess() throws AdapterException { assertTrue(testAdapter.calledStop); } + private TestAdapter getTestAdapterInstance() { + SpecificAdapterSetDescription description = new SpecificAdapterSetDescription(); + description.setRules(new ArrayList<>()); + TestAdapter testAdapter = new TestAdapter(description); + + return testAdapter; + } + private class TestAdapter extends SpecificDataSetAdapter { - public TestAdapter() { - super(null); + + public TestAdapter(SpecificAdapterSetDescription description) { + super(description); } public boolean calledStart = false; diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java index bf15c9997c..76c6f7b512 100644 --- a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java @@ -28,7 +28,6 @@ import org.streampipes.connect.management.master.GuessManagement; import org.streampipes.connect.utils.ConnectContainerResourceTest; import org.streampipes.connect.utils.Utils; -import org.streampipes.model.connect.guess.DomainPropertyProbabilityList; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventPropertyPrimitive; import org.streampipes.model.schema.EventSchema; @@ -38,11 +37,10 @@ import java.util.Arrays; import static com.jayway.restassured.RestAssured.given; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class GuessResourceTest extends ConnectContainerResourceTest { @@ -116,7 +114,7 @@ public void guessSchemaFail() throws AdapterException { .post(getApi() + "/schema"); res.then() - .statusCode(500); + .statusCode(200); } diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java index bbf299b28d..f4c445f87c 100644 --- a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java @@ -29,16 +29,12 @@ import org.streampipes.connect.management.master.SourcesManagement; import org.streampipes.connect.utils.ConnectContainerResourceTest; import org.streampipes.model.SpDataSet; -import org.streampipes.model.connect.grounding.FormatDescriptionList; import org.streampipes.model.graph.DataSourceDescription; import org.streampipes.rest.shared.util.JsonLdUtils; import static com.jayway.restassured.RestAssured.given; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; -import static org.powermock.api.mockito.PowerMockito.doNothing; public class SourcesResourceTest extends ConnectContainerResourceTest { @@ -141,40 +137,40 @@ public void getAdapterDataSourceFail() throws AdapterException { .statusCode(500); } - - @Test - public void addAdapterSuccess() throws Exception { - SourcesManagement sourcesManagement = mock(SourcesManagement.class); - doNothing().when(sourcesManagement).addAdapter(anyString(), anyString(), any()); - sourcesResource.setSourcesManagement(sourcesManagement); - - String data = getMinimalDataSetJsonLd(); - postJsonSuccessRequest(data, "/id/streams", "Instance of data set http://dataset.de/1 successfully started"); - - verify(sourcesManagement, times(1)).addAdapter(anyString(), anyString(), any()); - } - - @Test - public void addAdapterFail() throws AdapterException { - SourcesManagement sourcesManagement = mock(SourcesManagement.class); - doThrow(AdapterException.class).when(sourcesManagement).addAdapter(anyString(), anyString(), any()); - sourcesResource.setSourcesManagement(sourcesManagement); - - String data = getMinimalDataSetJsonLd(); - postJsonFailRequest(data, "/id/streams", "Could not set data set instance: http://dataset.de/1"); - - } - - @Test - public void detachSuccess() throws AdapterException { - SourcesManagement sourcesManagement = mock(SourcesManagement.class); - doNothing().when(sourcesManagement).detachAdapter(anyString(), anyString(), anyString()); - sourcesResource.setSourcesManagement(sourcesManagement); - - deleteJsonLdSucessRequest("/id0/streams/id1"); - - verify(sourcesManagement, times(1)).detachAdapter(anyString(), anyString(), anyString()); - } +// +// @Test +// public void addAdapterSuccess() throws Exception { +// SourcesManagement sourcesManagement = mock(SourcesManagement.class); +// doNothing().when(sourcesManagement).addAdapter(anyString(), anyString(), any()); +// sourcesResource.setSourcesManagement(sourcesManagement); +// +// String data = getMinimalDataSetJsonLd(); +// postJsonSuccessRequest(data, "/id/streams", "Instance of data set http://dataset.de/1 successfully started"); +// +// verify(sourcesManagement, times(1)).addAdapter(anyString(), anyString(), any()); +// } +// +// @Test +// public void addAdapterFail() throws AdapterException { +// SourcesManagement sourcesManagement = mock(SourcesManagement.class); +// doThrow(AdapterException.class).when(sourcesManagement).addAdapter(anyString(), anyString(), any()); +// sourcesResource.setSourcesManagement(sourcesManagement); +// +// String data = getMinimalDataSetJsonLd(); +// postJsonFailRequest(data, "/id/streams", "Could not set data set instance: http://dataset.de/1"); +// +// } + +// @Test +// public void detachSuccess() throws AdapterException { +// SourcesManagement sourcesManagement = mock(SourcesManagement.class); +// doNothing().when(sourcesManagement).detachAdapter(anyString(), anyString(), anyString()); +// sourcesResource.setSourcesManagement(sourcesManagement); +// +// deleteJsonLdSucessRequest("/id0/streams/id1"); +// +// verify(sourcesManagement, times(1)).detachAdapter(anyString(), anyString(), anyString()); +// } @Test public void detachFail() throws AdapterException { diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java index 07f93b5466..88a2feb88e 100644 --- a/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java @@ -23,6 +23,7 @@ import org.streampipes.vocabulary.StreamPipes; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; public class Utils { @@ -59,6 +60,7 @@ public static AdapterStreamDescription getMinimalStreamAdapter() { String id = "http://t.de/"; result.setUri(id); result.setId(id); + result.setRules(new ArrayList<>()); return result; } diff --git a/streampipes-connect/pom.xml b/streampipes-connect/pom.xml index 7c9426dc4e..e0bf425cd6 100755 --- a/streampipes-connect/pom.xml +++ b/streampipes-connect/pom.xml @@ -3,7 +3,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 @@ -104,6 +104,11 @@ org.apache.httpcomponents fluent-hc 4.5.5 + + + org.apache.camel + camel-milo + 2.22.1 org.apache.hadoop @@ -124,5 +129,15 @@ + + org.glassfish.jersey.media + jersey-media-sse + 2.22.2 + + + com.ullink.slack + simpleslackapi + 1.2.0 + \ No newline at end of file diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java index 76b9b2c90e..350d46663a 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java @@ -112,11 +112,11 @@ private AdapterPipeline getAdapterPipeline(T adapterDescription) { // Needed when adapter is if (adapterDescription.getEventGrounding() != null && adapterDescription.getEventGrounding().getTransportProtocol() != null - && adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname() != null) { + && adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname() != null) { pipelineElements.add(new SendToKafkaAdapterSink( adapterDescription)); } - return new AdapterPipeline(pipelineElements); + return new AdapterPipeline(pipelineElements); } private RemoveDuplicatesTransformationRuleDescription getRemoveDuplicateRule(T adapterDescription) { @@ -134,9 +134,11 @@ private AddValueTransformationRuleDescription getAddValueRule(T adapterDescripti private G getRule(T adapterDescription, Class type) { - for (TransformationRuleDescription tr : adapterDescription.getRules()) { - if (type.isInstance(tr)) { - return type.cast(tr); + if (adapterDescription != null) { + for (TransformationRuleDescription tr : adapterDescription.getRules()) { + if (type.isInstance(tr)) { + return type.cast(tr); + } } } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java index f630712bbf..bc6723b22d 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java @@ -43,8 +43,15 @@ import org.streampipes.connect.adapter.generic.protocol.stream.HttpStreamProtocol; import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; import org.streampipes.connect.adapter.generic.protocol.stream.MqttProtocol; +import org.streampipes.connect.adapter.specific.coindesk.CoindeskBitcoinAdapter; import org.streampipes.connect.adapter.specific.gdelt.GdeltAdapter; +import org.streampipes.connect.adapter.specific.iex.IexCloudNewsAdapter; +import org.streampipes.connect.adapter.specific.iex.IexCloudStockAdapter; +import org.streampipes.connect.adapter.specific.opcua.OpcUaAdapter; import org.streampipes.connect.adapter.specific.ros.RosBridgeAdapter; +import org.streampipes.connect.adapter.specific.slack.SlackAdapter; +import org.streampipes.connect.adapter.specific.wikipedia.WikipediaEditedArticlesAdapter; +import org.streampipes.connect.adapter.specific.wikipedia.WikipediaNewArticlesAdapter; import org.streampipes.model.connect.adapter.AdapterDescription; import java.util.HashMap; @@ -62,8 +69,15 @@ public static Map getAllAdapters() { allAdapters.put(GenericDataStreamAdapter.ID, new GenericDataStreamAdapter()); //allAdapters.put(OpenSenseMapAdapter.ID, new OpenSenseMapAdapter()); allAdapters.put(GdeltAdapter.ID, new GdeltAdapter()); + allAdapters.put(OpcUaAdapter.ID, new OpcUaAdapter()); //allAdapters.put(NswTrafficCameraAdapter.ID, new NswTrafficCameraAdapter()); allAdapters.put(RosBridgeAdapter.ID, new RosBridgeAdapter()); + allAdapters.put(CoindeskBitcoinAdapter.ID, new CoindeskBitcoinAdapter()); + allAdapters.put(IexCloudStockAdapter.ID, new IexCloudStockAdapter()); + allAdapters.put(IexCloudNewsAdapter.ID, new IexCloudNewsAdapter()); + allAdapters.put(WikipediaEditedArticlesAdapter.ID, new WikipediaEditedArticlesAdapter()); + allAdapters.put(WikipediaNewArticlesAdapter.ID, new WikipediaNewArticlesAdapter()); + allAdapters.put(SlackAdapter.ID, new SlackAdapter()); return allAdapters; } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageFormat.java index 0d63235827..0a2dedfd32 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageFormat.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageFormat.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -62,9 +61,6 @@ public static void main(String... args) throws IOException { .asStream(); byte[] b = IOUtils.toByteArray(result); - System.out.println(Base64.getEncoder().encodeToString(b)); - - System.out.println("========k=======k=======k=======k=======k=======k=======k=======k=======k=======k=======k=======k=======k======k"); // InputStream in = IOUtils.toInputStream(result, "UTF-8"); // byte[] a = IOUtils.toByteArray(in); @@ -85,8 +81,6 @@ public Map parse(byte[] object) throws ParseException { String resultImage = Base64.getEncoder().encodeToString(object); - System.out.println("Format " + Base64.getEncoder().encodeToString(object)); - result.put("image", resultImage); return result; diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageParser.java index ca70ec8f31..ced805b426 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageParser.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/image/ImageParser.java @@ -52,7 +52,6 @@ public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws Pars try { byte[] result = IOUtils.toByteArray(data); - System.out.println("Parser " + result.toString()); emitBinaryEvent.emit(result); } catch (IOException e) { throw new ParseException(e.getMessage()); @@ -65,6 +64,7 @@ public EventSchema getEventSchema(List oneEvent) { EventSchema resultSchema = new EventSchema(); EventPropertyPrimitive p = new EventPropertyPrimitive(); p.setRuntimeName("image"); + p.setLabel("Image"); p.setRuntimeType(XSD._string.toString()); p.setDomainProperties(Arrays.asList(URI.create("https://image.com"))); diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java index ef757a4888..64ea253f76 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java @@ -51,7 +51,6 @@ public Map process(Map event) { try { if (event != null) { producer.publish(objectMapper.writeValueAsBytes(event)); - System.out.println("send to kafka: " + event); } } catch (JsonProcessingException e) { e.printStackTrace(); diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java index b270313f49..c589362bbd 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java @@ -28,9 +28,8 @@ import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; import org.streampipes.connect.adapter.generic.protocol.Protocol; import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; -import org.streampipes.connect.exception.AdapterException; import org.streampipes.connect.exception.ParseException; -import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -67,11 +66,11 @@ public FileProtocol(Parser parser, Format format, String fileUri) { @Override public ProtocolDescription declareModel() { - return ProtocolDescriptionBuilder.create(ID, "File", "Reads the content from a local file.") + return ProtocolDescriptionBuilder.create(ID, "File Set", "Reads the content from a local file.") .sourceType(AdapterSourceType.SET) + .category(AdapterType.Generic) .iconUrl("file.png") - .requiredFile(Labels.from("filePath", "File", "This " + - "property defines the path to the file.")) + .requiredFile(Labels.from("filePath", "File", "File Path")) .build(); } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java index 7ef93e9027..ed1954e964 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java @@ -28,6 +28,7 @@ import org.streampipes.connect.adapter.generic.protocol.Protocol; import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -61,10 +62,10 @@ public HttpProtocol(Parser parser, Format format, String url) { public ProtocolDescription declareModel() { return ProtocolDescriptionBuilder.create(ID, "HTTP Set", "Reads the content from an HTTP " + "endpoint.") + .category(AdapterType.Generic) .sourceType(AdapterSourceType.SET) .iconUrl("rest.png") - .requiredTextParameter(Labels.from("url", "url", "This property defines the URL " + - "for the http request.")) + .requiredTextParameter(Labels.from("url", "Url", "Example: http(s)://test-server.com")) .build(); } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/FileStreamProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/FileStreamProtocol.java index 84673b6f74..c23a6468b3 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/FileStreamProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/FileStreamProtocol.java @@ -24,6 +24,7 @@ import org.streampipes.connect.adapter.generic.protocol.Protocol; import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -36,7 +37,6 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; -import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -93,13 +93,13 @@ public Protocol getInstance(ProtocolDescription protocolDescription, Parser pars @Override public ProtocolDescription declareModel() { - return ProtocolDescriptionBuilder.create(ID, "File", "Continuously streams the content of a " + + return ProtocolDescriptionBuilder.create(ID, "File Stream", "Continuously streams the content of a " + "file.") .sourceType(AdapterSourceType.STREAM) + .category(AdapterType.Generic) .iconUrl("file.png") - .requiredFile(Labels.from("filePath", "File", "This property defines the path to the file.")) - .requiredIntegerParameter(Labels.from("interval", "Interval", "This property " + - "defines the pull interval in seconds.")) + .requiredFile(Labels.from("filePath", "File", "File path")) + .requiredIntegerParameter(Labels.from("interval", "Interval", "Example: 5 (Polling interval in seconds)")) .build(); } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HDFSProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HDFSProtocol.java index 85bb5a1b7e..e831be0933 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HDFSProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HDFSProtocol.java @@ -32,6 +32,7 @@ import org.streampipes.connect.adapter.generic.protocol.Protocol; import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -111,12 +112,12 @@ public ProtocolDescription declareModel() { " System") .sourceType(AdapterSourceType.STREAM) .iconUrl("hdfs.png") - .requiredTextParameter(Labels.from(URL_PROPERTY, "HDFS-Server URL e.g. hdfs://server:8020", - "This property defines the HDFS URL e.g. hdfs://server:8020")) - .requiredIntegerParameter(Labels.from(INTERVAL_PROPERTY, "Interval", "This property " + - "defines the pull interval in seconds.")) + .category(AdapterType.Generic) + .requiredTextParameter(Labels.from(URL_PROPERTY, "HDFS-Server", + "Example: hdfs://server:8020")) + .requiredIntegerParameter(Labels.from(INTERVAL_PROPERTY, "Interval", "Polling interval in seconds")) .requiredTextParameter(Labels.from(DATA_PATH_PROPERTY, "Data Path", - "The Data Path which should be watched")) + "The Data Path to watch")) // .requiredTextParameter(Labels.from(USER_PROPERTY, "Username", "The Username to " + // "login")) // .requiredTextParameter(Labels.from(PASSWORD_PROPERTY, "Password","The Password to" + diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java index d6b1b6744f..0565165298 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java @@ -25,6 +25,7 @@ import org.streampipes.connect.adapter.generic.protocol.Protocol; import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -83,11 +84,10 @@ public ProtocolDescription declareModel() { return ProtocolDescriptionBuilder.create(ID, "HTTP Stream", "This is the " + "description for the http stream protocol") .sourceType(AdapterSourceType.STREAM) + .category(AdapterType.Generic) .iconUrl("rest.png") - .requiredTextParameter(Labels.from(URL_PROPERTY, "URL", "This property " + - "defines the URL for the http request.")) - .requiredIntegerParameter(Labels.from(INTERVAL_PROPERTY, "Interval", "This property " + - "defines the pull interval in seconds.")) + .requiredTextParameter(Labels.from(URL_PROPERTY, "Url", "Example: http(s)://test-server.com")) + .requiredIntegerParameter(Labels.from(INTERVAL_PROPERTY, "Interval", "Example: 5 (Polling interval in seconds)")) //.requiredTextParameter(Labels.from(ACCESS_TOKEN_PROPERTY, "Access Token", "Http // Access Token")) .build(); diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java index f2644c1ea0..b1fb0dac83 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java @@ -40,6 +40,7 @@ import org.streampipes.connect.exception.ParseException; import org.streampipes.messaging.InternalEventProcessor; import org.streampipes.messaging.kafka.SpKafkaConsumer; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.sdk.builder.adapter.ProtocolDescriptionBuilder; @@ -85,11 +86,12 @@ public ProtocolDescription declareModel() { return ProtocolDescriptionBuilder.create(ID,"Apache Kafka","Consumes messages from an " + "Apache Kafka broker") .iconUrl("kafka.jpg") + .category(AdapterType.Generic, AdapterType.Manufacturing) .sourceType(AdapterSourceType.STREAM) .requiredTextParameter(Labels.from("broker_url", "Broker URL", - "This property defines the URL of the Kafka broker.")) + "Example: test.server.com:9092 (No protocol. Port required)")) .requiredTextParameter(Labels.from("topic", "Topic", - "Topic in the broker")) + "Example: test.topic")) .build(); } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java index 01bcd0ec6c..5f13cf1978 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java @@ -28,10 +28,15 @@ public class MqttConsumer implements Runnable { private int maxElementsToReceive = -1; private int messageCount = 0; + private Boolean authenticatedConnection; + private String username; + private String password; + public MqttConsumer(String broker, String topic, InternalEventProcessor consumer) { this.broker = broker; this.topic = topic; this.consumer = consumer; + this.authenticatedConnection = false; } public MqttConsumer(String broker, String topic, InternalEventProcessor consumer, int maxElementsToReceive) { @@ -39,6 +44,14 @@ public MqttConsumer(String broker, String topic, InternalEventProcessor this.maxElementsToReceive = maxElementsToReceive; } + public MqttConsumer(String broker, String topic, String username, String password, + InternalEventProcessor consumer) { + this(broker, topic, consumer); + this.username = username; + this.password = password; + this.authenticatedConnection = true; + } + @Override public void run() { @@ -46,6 +59,10 @@ public void run() { MQTT mqtt = new MQTT(); try { mqtt.setHost(broker); + if (authenticatedConnection) { + mqtt.setUserName(username); + mqtt.setPassword(password); + } BlockingConnection connection = mqtt.blockingConnection(); connection.connect(); Topic[] topics = {new Topic(topic, QoS.AT_LEAST_ONCE)}; diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java index 4cccac1d74..c247fdd7ca 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java @@ -25,6 +25,7 @@ import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.connect.exception.ParseException; import org.streampipes.messaging.InternalEventProcessor; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.sdk.builder.adapter.ProtocolDescriptionBuilder; import org.streampipes.sdk.helpers.AdapterSourceType; @@ -62,10 +63,11 @@ public ProtocolDescription declareModel() { return ProtocolDescriptionBuilder.create(ID, "MQTT", "Consumes messages from a broker using " + "the MQTT protocol") .iconUrl("mqtt.png") + .category(AdapterType.Generic, AdapterType.Manufacturing) .sourceType(AdapterSourceType.STREAM) .requiredTextParameter(Labels.from("broker_url", "Broker URL", - "This property defines the URL of the MQTT broker.")) - .requiredTextParameter(Labels.from("topic", "Topic","The topic to subscribe to")) + "Example: tcp://test-server.com:1883 (Protocol required. Port required)")) + .requiredTextParameter(Labels.from("topic", "Topic","Example: test/topic")) .build(); } diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java index 97d499f621..d817132848 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java @@ -43,6 +43,10 @@ public List selectedMultiValues(String internalName) { .collect(Collectors.toList()); } + public String selectedSingleValueOption(String internalName) { + return selectedMultiValues(internalName).get(0); + } + public StaticProperty getStaticPropertyByName(String name) { for(StaticProperty p : list) diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/CoindeskBitcoinAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/CoindeskBitcoinAdapter.java new file mode 100644 index 0000000000..a57ea86640 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/CoindeskBitcoinAdapter.java @@ -0,0 +1,122 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.coindesk; + +import com.google.gson.Gson; +import org.apache.http.client.fluent.Request; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.connect.adapter.specific.PullAdapter; +import org.streampipes.connect.adapter.specific.coindesk.model.CoindeskRawModel; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.builder.adapter.GuessSchemaBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.EpProperties; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.helpers.Options; +import org.streampipes.vocabulary.SO; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class CoindeskBitcoinAdapter extends PullAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/coindesk/bitcoin"; + + private static final String CoindeskUrl = "https://api.coindesk.com/v1/bpi/currentprice.json"; + + private Currency currency; + + public CoindeskBitcoinAdapter() { + + } + + public CoindeskBitcoinAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + ParameterExtractor extractor = new ParameterExtractor(adapterDescription.getConfig()); + this.currency = Currency.valueOf(extractor.selectedSingleValueOption("currency")); + + } + + @Override + protected void pullData() { + try { + String response = Request.Get(CoindeskUrl).execute().returnContent().asString(); + CoindeskRawModel rawModel = new Gson().fromJson(response, CoindeskRawModel.class); + + long timestamp = System.currentTimeMillis(); + Double rate; + if (currency == Currency.EUR) { + rate = rawModel.getBpi().getEUR().getRateFloat(); + } else if (currency == Currency.GBP) { + rate = rawModel.getBpi().getGBP().getRateFloat(); + } else { + rate = rawModel.getBpi().getUSD().getRateFloat(); + } + + Map outMap = new HashMap<>(); + outMap.put("timestamp", timestamp); + outMap.put("rate", rate); + + adapterPipeline.process(outMap); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + protected PollingSettings getPollingInterval() { + return PollingSettings.from(TimeUnit.SECONDS, 60); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "Coindesk Bitcoin Stream", "The current " + + "bitcoin price from the Coindesk API.") + .iconUrl("coindesk.png") + .category(AdapterType.Finance) + .requiredSingleValueSelection(Labels.from("currency", "Currency", "The currency of the" + + " bitcoin rate"), Options.from("USD", "EUR", "GBP")) + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new CoindeskBitcoinAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + return GuessSchemaBuilder.create() + .property(EpProperties.timestampProperty("timestamp")) + .property(EpProperties.doubleEp(Labels.from("rate-field", "Rate", "The current " + + "bitcoin rate"), "rate", SO.Price)) + .build(); + } + + @Override + public String getId() { + return ID; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/Currency.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/Currency.java new file mode 100644 index 0000000000..d6b07dab29 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/Currency.java @@ -0,0 +1,24 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.specific.coindesk; + +public enum Currency { + EUR, + GBP, + USD; + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Bpi.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Bpi.java new file mode 100644 index 0000000000..48ea5a467d --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Bpi.java @@ -0,0 +1,42 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Bpi { + + @SerializedName("EUR") + private EUR mEUR; + @SerializedName("GBP") + private GBP mGBP; + @SerializedName("USD") + private USD mUSD; + + public EUR getEUR() { + return mEUR; + } + + public void setEUR(EUR eUR) { + mEUR = eUR; + } + + public GBP getGBP() { + return mGBP; + } + + public void setGBP(GBP gBP) { + mGBP = gBP; + } + + public USD getUSD() { + return mUSD; + } + + public void setUSD(USD uSD) { + mUSD = uSD; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/CoindeskRawModel.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/CoindeskRawModel.java new file mode 100644 index 0000000000..f0abfbb287 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/CoindeskRawModel.java @@ -0,0 +1,52 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class CoindeskRawModel { + + @SerializedName("bpi") + private Bpi mBpi; + @SerializedName("chartName") + private String mChartName; + @SerializedName("disclaimer") + private String mDisclaimer; + @SerializedName("time") + private Time mTime; + + public Bpi getBpi() { + return mBpi; + } + + public void setBpi(Bpi bpi) { + mBpi = bpi; + } + + public String getChartName() { + return mChartName; + } + + public void setChartName(String chartName) { + mChartName = chartName; + } + + public String getDisclaimer() { + return mDisclaimer; + } + + public void setDisclaimer(String disclaimer) { + mDisclaimer = disclaimer; + } + + public Time getTime() { + return mTime; + } + + public void setTime(Time time) { + mTime = time; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/EUR.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/EUR.java new file mode 100644 index 0000000000..5ffb3d39c6 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/EUR.java @@ -0,0 +1,62 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class EUR { + + @SerializedName("code") + private String mCode; + @SerializedName("description") + private String mDescription; + @SerializedName("rate") + private String mRate; + @SerializedName("rate_float") + private Double mRateFloat; + @SerializedName("symbol") + private String mSymbol; + + public String getCode() { + return mCode; + } + + public void setCode(String code) { + mCode = code; + } + + public String getDescription() { + return mDescription; + } + + public void setDescription(String description) { + mDescription = description; + } + + public String getRate() { + return mRate; + } + + public void setRate(String rate) { + mRate = rate; + } + + public Double getRateFloat() { + return mRateFloat; + } + + public void setRateFloat(Double rateFloat) { + mRateFloat = rateFloat; + } + + public String getSymbol() { + return mSymbol; + } + + public void setSymbol(String symbol) { + mSymbol = symbol; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/GBP.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/GBP.java new file mode 100644 index 0000000000..c871423290 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/GBP.java @@ -0,0 +1,62 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class GBP { + + @SerializedName("code") + private String mCode; + @SerializedName("description") + private String mDescription; + @SerializedName("rate") + private String mRate; + @SerializedName("rate_float") + private Double mRateFloat; + @SerializedName("symbol") + private String mSymbol; + + public String getCode() { + return mCode; + } + + public void setCode(String code) { + mCode = code; + } + + public String getDescription() { + return mDescription; + } + + public void setDescription(String description) { + mDescription = description; + } + + public String getRate() { + return mRate; + } + + public void setRate(String rate) { + mRate = rate; + } + + public Double getRateFloat() { + return mRateFloat; + } + + public void setRateFloat(Double rateFloat) { + mRateFloat = rateFloat; + } + + public String getSymbol() { + return mSymbol; + } + + public void setSymbol(String symbol) { + mSymbol = symbol; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Time.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Time.java new file mode 100644 index 0000000000..89d4544fcd --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/Time.java @@ -0,0 +1,42 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Time { + + @SerializedName("updated") + private String mUpdated; + @SerializedName("updatedISO") + private String mUpdatedISO; + @SerializedName("updateduk") + private String mUpdateduk; + + public String getUpdated() { + return mUpdated; + } + + public void setUpdated(String updated) { + mUpdated = updated; + } + + public String getUpdatedISO() { + return mUpdatedISO; + } + + public void setUpdatedISO(String updatedISO) { + mUpdatedISO = updatedISO; + } + + public String getUpdateduk() { + return mUpdateduk; + } + + public void setUpdateduk(String updateduk) { + mUpdateduk = updateduk; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/USD.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/USD.java new file mode 100644 index 0000000000..c121c86f69 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/coindesk/model/USD.java @@ -0,0 +1,62 @@ + +package org.streampipes.connect.adapter.specific.coindesk.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class USD { + + @SerializedName("code") + private String mCode; + @SerializedName("description") + private String mDescription; + @SerializedName("rate") + private String mRate; + @SerializedName("rate_float") + private Double mRateFloat; + @SerializedName("symbol") + private String mSymbol; + + public String getCode() { + return mCode; + } + + public void setCode(String code) { + mCode = code; + } + + public String getDescription() { + return mDescription; + } + + public void setDescription(String description) { + mDescription = description; + } + + public String getRate() { + return mRate; + } + + public void setRate(String rate) { + mRate = rate; + } + + public Double getRateFloat() { + return mRateFloat; + } + + public void setRateFloat(Double rateFloat) { + mRateFloat = rateFloat; + } + + public String getSymbol() { + return mSymbol; + } + + public void setSymbol(String symbol) { + mSymbol = symbol; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudAdapter.java new file mode 100644 index 0000000000..838c292ffd --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudAdapter.java @@ -0,0 +1,53 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.iex; + +import com.google.gson.Gson; +import org.apache.http.client.fluent.Request; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.connect.adapter.specific.PullAdapter; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; + +import java.io.IOException; + +public abstract class IexCloudAdapter extends PullAdapter { + + protected static final String IexCloudBaseUrl = "https://cloud.iexapis.com/stable/stock/"; + protected static final String Token = "?token="; + + protected String apiToken; + protected String stockQuote; + private String iexCloudInstanceUrl; + + + public IexCloudAdapter(SpecificAdapterStreamDescription adapterDescription, String restPath) { + super(adapterDescription); + ParameterExtractor extractor = new ParameterExtractor(adapterDescription.getConfig()); + this.apiToken = extractor.singleValue("token"); + this.stockQuote = extractor.singleValue("stock"); + this.iexCloudInstanceUrl = IexCloudBaseUrl + stockQuote + restPath + Token + apiToken; + + } + + public IexCloudAdapter() { + super(); + } + + protected T fetchResult(Class classToParse) throws IOException { + String response = Request.Get(iexCloudInstanceUrl).execute().returnContent().asString(); + return new Gson().fromJson(response, classToParse); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudNewsAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudNewsAdapter.java new file mode 100644 index 0000000000..f97a7ec90f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudNewsAdapter.java @@ -0,0 +1,134 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.iex; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.iex.model.IexNewsData; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.builder.adapter.GuessSchemaBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.EpProperties; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.vocabulary.SO; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class IexCloudNewsAdapter extends IexCloudAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/iexcloud/news"; + private static final String News = "/news"; + + private static final String Timestamp = "timestamp"; + private static final String Headline = "headline"; + private static final String Source = "source"; + private static final String Url = "url"; + private static final String Summary = "summary"; + private static final String Related = "related"; + private static final String Image = "image"; + private static final String Lang = "lang"; + private static final String HasPaywall = "hasPaywall"; + + public IexCloudNewsAdapter(SpecificAdapterStreamDescription adapterStreamDescription) { + super(adapterStreamDescription, News); + } + + public IexCloudNewsAdapter() { + super(); + } + + @Override + protected void pullData() { + try { + IexNewsData[] rawModel = fetchResult(IexNewsData[].class); + + for (IexNewsData newsData : rawModel) { + Map outMap = new HashMap<>(); + outMap.put(Timestamp, newsData.getDatetime()); + outMap.put(Headline, newsData.getHeadline()); + outMap.put(Source, newsData.getSource()); + outMap.put(Url, newsData.getUrl()); + outMap.put(Summary, newsData.getSummary()); + outMap.put(Related, newsData.getRelated()); + outMap.put(Image, newsData.getImage()); + outMap.put(Lang, newsData.getLang()); + outMap.put(HasPaywall, newsData.getHasPaywall()); + + adapterPipeline.process(outMap); + } + } catch ( + IOException e) { + e.printStackTrace(); + } + } + + @Override + protected PollingSettings getPollingInterval() { + return PollingSettings.from(TimeUnit.SECONDS, 60); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "IEX Cloud News", "Fetches news for a " + + "given company (10 news / minutes maximum)") + .iconUrl("iexcloud.png") + .category(AdapterType.Finance, AdapterType.News) + .requiredTextParameter(Labels.from("token", "API Token", "The IEXCloud API token")) + .requiredTextParameter(Labels.from("stock", "Stock", "The stock symbol (e.g., AAPL)")) + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new IexCloudNewsAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + return GuessSchemaBuilder.create() + .property(EpProperties.timestampProperty(Timestamp)) + .property(EpProperties.stringEp(Labels.from("headline", "Headline", + "The headline of the article"), Headline, SO.Text)) + .property(EpProperties.stringEp(Labels.from("source", "Source", + "The source of the article"), Source, SO.Text)) + .property(EpProperties.stringEp(Labels.from("url", "URL", + "The URL of the article"), Url, SO.ContentUrl)) + .property(EpProperties.stringEp(Labels.from("summary", "Summary", + "A short summary of the article"), Summary, SO.Text)) + .property(EpProperties.stringEp(Labels.from("related", "Related", + "A comma-separated list of related stock symbols"), Related, SO.Text)) + .property(EpProperties.stringEp(Labels.from("image", "Image", + "Link to an image related to the news article"), Image, SO.Image)) + .property(EpProperties.stringEp(Labels.from("lang", "Language", + "The language the article is writte in"), Lang, SO.InLanguage)) + .property(EpProperties.stringEp(Labels.from("paywall", "Has Paywall", + "Indicates whether the article is behind a paywall"), HasPaywall, + SO.Text)) + .build(); + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudStockAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudStockAdapter.java new file mode 100644 index 0000000000..3d90f45f44 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/IexCloudStockAdapter.java @@ -0,0 +1,108 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.iex; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.iex.model.IexStockData; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.builder.adapter.GuessSchemaBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.EpProperties; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.vocabulary.SO; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class IexCloudStockAdapter extends IexCloudAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/iexcloud/stocks"; + + private static final String Quotes = "/quote"; + private static final String LatestUpdate = "latestUpdate"; + private static final String LatestPrice = "latestPrice"; + private static final String Symbol = "symbol"; + + public IexCloudStockAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription, Quotes); + } + + public IexCloudStockAdapter() { + super(); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "IEX Cloud Stock Quotes", "Live stock data" + + " provided by IEX Cloud") + .iconUrl("iexcloud.png") + .category(AdapterType.Finance) + .requiredTextParameter(Labels.from("token", "API Token", "The IEXCloud API token")) + .requiredTextParameter(Labels.from("stock", "Stock", "The stock symbol (e.g., AAPL)")) + .build(); + + } + + @Override + protected void pullData() { + try { + IexStockData rawModel = fetchResult(IexStockData.class); + + Map outMap = new HashMap<>(); + outMap.put(LatestUpdate, rawModel.getLatestUpdate()); + outMap.put(Symbol, rawModel.getSymbol()); + outMap.put(LatestPrice, rawModel.getLatestPrice()); + + adapterPipeline.process(outMap); + } catch ( + IOException e) { + e.printStackTrace(); + } + } + + @Override + protected PollingSettings getPollingInterval() { + return PollingSettings.from(TimeUnit.SECONDS, 5); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new IexCloudStockAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + return GuessSchemaBuilder.create() + .property(EpProperties.timestampProperty(LatestUpdate)) + .property(EpProperties.stringEp(Labels.from("symbol", "Symbol", + "The stock symbol"), Symbol, SO.Text)) + .property(EpProperties.doubleEp(Labels.from("latest-price", "Latest price", + "The latest stock price"), LatestPrice, SO.Number)) + .build(); + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexNewsData.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexNewsData.java new file mode 100644 index 0000000000..393f5b0bab --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexNewsData.java @@ -0,0 +1,102 @@ + +package org.streampipes.connect.adapter.specific.iex.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class IexNewsData { + + @SerializedName("datetime") + private Long mDatetime; + @SerializedName("hasPaywall") + private Boolean mHasPaywall; + @SerializedName("headline") + private String mHeadline; + @SerializedName("image") + private String mImage; + @SerializedName("lang") + private String mLang; + @SerializedName("related") + private String mRelated; + @SerializedName("source") + private String mSource; + @SerializedName("summary") + private String mSummary; + @SerializedName("url") + private String mUrl; + + public Long getDatetime() { + return mDatetime; + } + + public void setDatetime(Long datetime) { + mDatetime = datetime; + } + + public Boolean getHasPaywall() { + return mHasPaywall; + } + + public void setHasPaywall(Boolean hasPaywall) { + mHasPaywall = hasPaywall; + } + + public String getHeadline() { + return mHeadline; + } + + public void setHeadline(String headline) { + mHeadline = headline; + } + + public String getImage() { + return mImage; + } + + public void setImage(String image) { + mImage = image; + } + + public String getLang() { + return mLang; + } + + public void setLang(String lang) { + mLang = lang; + } + + public String getRelated() { + return mRelated; + } + + public void setRelated(String related) { + mRelated = related; + } + + public String getSource() { + return mSource; + } + + public void setSource(String source) { + mSource = source; + } + + public String getSummary() { + return mSummary; + } + + public void setSummary(String summary) { + mSummary = summary; + } + + public String getUrl() { + return mUrl; + } + + public void setUrl(String url) { + mUrl = url; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexStockData.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexStockData.java new file mode 100644 index 0000000000..d136f68056 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/iex/model/IexStockData.java @@ -0,0 +1,393 @@ + +package org.streampipes.connect.adapter.specific.iex.model; + +import com.google.gson.annotations.SerializedName; + +import javax.annotation.Generated; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class IexStockData { + + @SerializedName("avgTotalVolume") + private Long mAvgTotalVolume; + @SerializedName("calculationPrice") + private String mCalculationPrice; + @SerializedName("change") + private Double mChange; + @SerializedName("changePercent") + private Double mChangePercent; + @SerializedName("close") + private Double mClose; + @SerializedName("closeTime") + private Long mCloseTime; + @SerializedName("companyName") + private String mCompanyName; + @SerializedName("delayedPrice") + private Double mDelayedPrice; + @SerializedName("delayedPriceTime") + private Long mDelayedPriceTime; + @SerializedName("extendedChange") + private Double mExtendedChange; + @SerializedName("extendedChangePercent") + private Double mExtendedChangePercent; + @SerializedName("extendedPrice") + private Double mExtendedPrice; + @SerializedName("extendedPriceTime") + private Long mExtendedPriceTime; + @SerializedName("high") + private Double mHigh; + @SerializedName("iexAskPrice") + private Long mIexAskPrice; + @SerializedName("iexAskSize") + private Long mIexAskSize; + @SerializedName("iexBidPrice") + private Long mIexBidPrice; + @SerializedName("iexBidSize") + private Long mIexBidSize; + @SerializedName("iexLastUpdated") + private Long mIexLastUpdated; + @SerializedName("iexMarketPercent") + private Double mIexMarketPercent; + @SerializedName("iexRealtimePrice") + private Double mIexRealtimePrice; + @SerializedName("iexRealtimeSize") + private Long mIexRealtimeSize; + @SerializedName("iexVolume") + private Long mIexVolume; + @SerializedName("latestPrice") + private Double mLatestPrice; + @SerializedName("latestSource") + private String mLatestSource; + @SerializedName("latestTime") + private String mLatestTime; + @SerializedName("latestUpdate") + private Long mLatestUpdate; + @SerializedName("latestVolume") + private Long mLatestVolume; + @SerializedName("low") + private Double mLow; + @SerializedName("marketCap") + private Long mMarketCap; + @SerializedName("open") + private Double mOpen; + @SerializedName("openTime") + private Long mOpenTime; + @SerializedName("peRatio") + private Double mPeRatio; + @SerializedName("previousClose") + private Double mPreviousClose; + @SerializedName("symbol") + private String mSymbol; + @SerializedName("week52High") + private Double mWeek52High; + @SerializedName("week52Low") + private Long mWeek52Low; + @SerializedName("ytdChange") + private Double mYtdChange; + + public Long getAvgTotalVolume() { + return mAvgTotalVolume; + } + + public void setAvgTotalVolume(Long avgTotalVolume) { + mAvgTotalVolume = avgTotalVolume; + } + + public String getCalculationPrice() { + return mCalculationPrice; + } + + public void setCalculationPrice(String calculationPrice) { + mCalculationPrice = calculationPrice; + } + + public Double getChange() { + return mChange; + } + + public void setChange(Double change) { + mChange = change; + } + + public Double getChangePercent() { + return mChangePercent; + } + + public void setChangePercent(Double changePercent) { + mChangePercent = changePercent; + } + + public Double getClose() { + return mClose; + } + + public void setClose(Double close) { + mClose = close; + } + + public Long getCloseTime() { + return mCloseTime; + } + + public void setCloseTime(Long closeTime) { + mCloseTime = closeTime; + } + + public String getCompanyName() { + return mCompanyName; + } + + public void setCompanyName(String companyName) { + mCompanyName = companyName; + } + + public Double getDelayedPrice() { + return mDelayedPrice; + } + + public void setDelayedPrice(Double delayedPrice) { + mDelayedPrice = delayedPrice; + } + + public Long getDelayedPriceTime() { + return mDelayedPriceTime; + } + + public void setDelayedPriceTime(Long delayedPriceTime) { + mDelayedPriceTime = delayedPriceTime; + } + + public Double getExtendedChange() { + return mExtendedChange; + } + + public void setExtendedChange(Double extendedChange) { + mExtendedChange = extendedChange; + } + + public Double getExtendedChangePercent() { + return mExtendedChangePercent; + } + + public void setExtendedChangePercent(Double extendedChangePercent) { + mExtendedChangePercent = extendedChangePercent; + } + + public Double getExtendedPrice() { + return mExtendedPrice; + } + + public void setExtendedPrice(Double extendedPrice) { + mExtendedPrice = extendedPrice; + } + + public Long getExtendedPriceTime() { + return mExtendedPriceTime; + } + + public void setExtendedPriceTime(Long extendedPriceTime) { + mExtendedPriceTime = extendedPriceTime; + } + + public Double getHigh() { + return mHigh; + } + + public void setHigh(Double high) { + mHigh = high; + } + + public Long getIexAskPrice() { + return mIexAskPrice; + } + + public void setIexAskPrice(Long iexAskPrice) { + mIexAskPrice = iexAskPrice; + } + + public Long getIexAskSize() { + return mIexAskSize; + } + + public void setIexAskSize(Long iexAskSize) { + mIexAskSize = iexAskSize; + } + + public Long getIexBidPrice() { + return mIexBidPrice; + } + + public void setIexBidPrice(Long iexBidPrice) { + mIexBidPrice = iexBidPrice; + } + + public Long getIexBidSize() { + return mIexBidSize; + } + + public void setIexBidSize(Long iexBidSize) { + mIexBidSize = iexBidSize; + } + + public Long getIexLastUpdated() { + return mIexLastUpdated; + } + + public void setIexLastUpdated(Long iexLastUpdated) { + mIexLastUpdated = iexLastUpdated; + } + + public Double getIexMarketPercent() { + return mIexMarketPercent; + } + + public void setIexMarketPercent(Double iexMarketPercent) { + mIexMarketPercent = iexMarketPercent; + } + + public Double getIexRealtimePrice() { + return mIexRealtimePrice; + } + + public void setIexRealtimePrice(Double iexRealtimePrice) { + mIexRealtimePrice = iexRealtimePrice; + } + + public Long getIexRealtimeSize() { + return mIexRealtimeSize; + } + + public void setIexRealtimeSize(Long iexRealtimeSize) { + mIexRealtimeSize = iexRealtimeSize; + } + + public Long getIexVolume() { + return mIexVolume; + } + + public void setIexVolume(Long iexVolume) { + mIexVolume = iexVolume; + } + + public Double getLatestPrice() { + return mLatestPrice; + } + + public void setLatestPrice(Double latestPrice) { + mLatestPrice = latestPrice; + } + + public String getLatestSource() { + return mLatestSource; + } + + public void setLatestSource(String latestSource) { + mLatestSource = latestSource; + } + + public String getLatestTime() { + return mLatestTime; + } + + public void setLatestTime(String latestTime) { + mLatestTime = latestTime; + } + + public Long getLatestUpdate() { + return mLatestUpdate; + } + + public void setLatestUpdate(Long latestUpdate) { + mLatestUpdate = latestUpdate; + } + + public Long getLatestVolume() { + return mLatestVolume; + } + + public void setLatestVolume(Long latestVolume) { + mLatestVolume = latestVolume; + } + + public Double getLow() { + return mLow; + } + + public void setLow(Double low) { + mLow = low; + } + + public Long getMarketCap() { + return mMarketCap; + } + + public void setMarketCap(Long marketCap) { + mMarketCap = marketCap; + } + + public Double getOpen() { + return mOpen; + } + + public void setOpen(Double open) { + mOpen = open; + } + + public Long getOpenTime() { + return mOpenTime; + } + + public void setOpenTime(Long openTime) { + mOpenTime = openTime; + } + + public Double getPeRatio() { + return mPeRatio; + } + + public void setPeRatio(Double peRatio) { + mPeRatio = peRatio; + } + + public Double getPreviousClose() { + return mPreviousClose; + } + + public void setPreviousClose(Double previousClose) { + mPreviousClose = previousClose; + } + + public String getSymbol() { + return mSymbol; + } + + public void setSymbol(String symbol) { + mSymbol = symbol; + } + + public Double getWeek52High() { + return mWeek52High; + } + + public void setWeek52High(Double week52High) { + mWeek52High = week52High; + } + + public Long getWeek52Low() { + return mWeek52Low; + } + + public void setWeek52Low(Long week52Low) { + mWeek52Low = week52Low; + } + + public Double getYtdChange() { + return mYtdChange; + } + + public void setYtdChange(Double ytdChange) { + mYtdChange = ytdChange; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java index 12b7c28730..a25a927606 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java @@ -21,6 +21,7 @@ import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model.FeatureCollection; import org.streampipes.connect.adapter.specific.sensemap.SensorNames; import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventProperty; @@ -85,6 +86,7 @@ private List> getEvents() { public SpecificAdapterStreamDescription declareModel() { SpecificAdapterStreamDescription description = SpecificDataStreamAdapterBuilder.create(ID, "NSW Traffic Cameras", "Traffic camera " + "images produced by NSW Australia") + .category(AdapterType.OpenData) .requiredTextParameter(Labels.from("api-key", "API Key", "The TfNSW " + "API key")) .iconUrl("nsw.png") diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUa.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUa.java new file mode 100644 index 0000000000..a521a469b4 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUa.java @@ -0,0 +1,218 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific.opcua; + + +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.api.config.OpcUaClientConfig; +import org.eclipse.milo.opcua.sdk.client.api.subscriptions.UaMonitoredItem; +import org.eclipse.milo.opcua.sdk.client.api.subscriptions.UaSubscription; +import org.eclipse.milo.opcua.stack.client.UaTcpStackClient; +import org.eclipse.milo.opcua.stack.core.AttributeId; +import org.eclipse.milo.opcua.stack.core.Identifiers; +import org.eclipse.milo.opcua.stack.core.security.SecurityPolicy; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.LocalizedText; +import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; +import org.eclipse.milo.opcua.stack.core.types.builtin.QualifiedName; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.enumerated.*; +import org.eclipse.milo.opcua.stack.core.types.structured.*; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.eclipse.milo.opcua.stack.core.util.ConversionUtil.toList; + +public class OpcUa { + + private NodeId node; + private String opcServerHost; + private int opcServerPort; + private OpcUaClient client; + + private static final AtomicLong clientHandles = new AtomicLong(1L); + + public OpcUa(String opcServerURL, int opcServerPort, int namespaceIndex, String nodeId) { + + this.opcServerHost = opcServerURL; + this.opcServerPort = opcServerPort; + this.node = new NodeId(namespaceIndex, nodeId); + + } + + public void connect() throws Exception { + + EndpointDescription[] endpoints = UaTcpStackClient.getEndpoints("opc.tcp://" + opcServerHost + ":" + opcServerPort).get(); + + EndpointDescription tmpEndpoint = endpoints[0]; + tmpEndpoint = updateEndpointUrl(tmpEndpoint, opcServerHost); + endpoints = new EndpointDescription[]{tmpEndpoint}; + + EndpointDescription endpoint = Arrays.stream(endpoints) + .filter(e -> e.getSecurityPolicyUri().equals(SecurityPolicy.None.getSecurityPolicyUri())) + .findFirst().orElseThrow(() -> new Exception("no desired endpoints returned")); + + OpcUaClientConfig config = OpcUaClientConfig.builder() + .setApplicationName(LocalizedText.english("eclipse milo opc-ua client")) + .setApplicationUri("urn:eclipse:milo:examples:client") + .setEndpoint(endpoint) + .build(); + + this.client = new OpcUaClient(config); + client.connect().get(); + } + + public void disconnect() { + client.disconnect(); + } + + private EndpointDescription updateEndpointUrl( + EndpointDescription original, String hostname) throws URISyntaxException { + + URI uri = new URI(original.getEndpointUrl()).parseServerAuthority(); + + String endpointUrl = String.format( + "%s://%s:%s%s", + uri.getScheme(), + hostname, + uri.getPort(), + uri.getPath() + ); + + return new EndpointDescription( + endpointUrl, + original.getServer(), + original.getServerCertificate(), + original.getSecurityMode(), + original.getSecurityPolicyUri(), + original.getUserIdentityTokens(), + original.getTransportProfileUri(), + original.getSecurityLevel() + ); + } + + public List browseNode() { + return browseNode(node); + } + + private List browseNode(NodeId browseRoot) { + List result = new ArrayList<>(); + + BrowseDescription browse = new BrowseDescription( + browseRoot, + BrowseDirection.Forward, + Identifiers.References, + true, + uint(NodeClass.Object.getValue() | NodeClass.Variable.getValue()), + uint(BrowseResultMask.All.getValue()) + ); + + try { + BrowseResult browseResult = client.browse(browse).get(); + + List references = toList(browseResult.getReferences()); + + for (ReferenceDescription rd : references) { + result.add(rd); + rd.getNodeId().local().ifPresent(nodeId -> browseNode(nodeId)); + } + } catch (InterruptedException | ExecutionException e) { + System.out.println("Browsing nodeId=" + browse + " failed: " + e.getMessage()); + } + + return result; + + } + + + public void createListSubscription(List nodes, OpcUaAdapter opcUaAdapter) throws Exception { + /* + * create a subscription @ 1000ms + */ + UaSubscription subscription = this.client.getSubscriptionManager().createSubscription(1000.0).get(); + + + List> values = new ArrayList<>(); + + for (NodeId node : nodes) { + values.add(this.client.readValue(0, TimestampsToReturn.Both, node)); + } + + for (CompletableFuture value : values) { + if (value.get().getValue().toString().contains("null")) { + System.out.println("Node has no value"); + } + } + + + List readValues = new ArrayList<>(); + // Read a specific value attribute + for (NodeId node : nodes) { + readValues.add(new ReadValueId(node, AttributeId.Value.uid(), null, QualifiedName.NULL_VALUE)); + } + + List requests = new ArrayList<>(); + + for (ReadValueId readValue : readValues) { + // important: client handle must be unique per item + UInteger clientHandle = uint(clientHandles.getAndIncrement()); + + MonitoringParameters parameters = new MonitoringParameters( + clientHandle, + 1000.0, // sampling interval + null, // filter, null means use default + uint(10), // queue size + true // discard oldest + ); + + requests.add(new MonitoredItemCreateRequest(readValue, MonitoringMode.Reporting, parameters)); + } + + BiConsumer onItemCreated = + (item, id) -> { + item.setValueConsumer(opcUaAdapter::onSubscriptionValue); + }; + + List items = subscription.createMonitoredItems( + TimestampsToReturn.Both, + requests, + onItemCreated + ).get(); + + for (UaMonitoredItem item : items) { + NodeId tagId = item.getReadValueId().getNodeId(); + if (item.getStatusCode().isGood()) { + System.out.println("item created for nodeId="+ tagId); + } else { + System.out.println("failed to create item for " + item.getReadValueId().getNodeId() + item.getStatusCode()); + } + } + + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaAdapter.java new file mode 100644 index 0000000000..e022c906f8 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaAdapter.java @@ -0,0 +1,202 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific.opcua; + +import org.eclipse.milo.opcua.sdk.client.api.subscriptions.UaMonitoredItem; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.structured.ReferenceDescription; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.SpecificDataStreamAdapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.streampipes.model.staticproperty.StaticProperty; +import org.streampipes.sdk.builder.PrimitivePropertyBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.Labels; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class OpcUaAdapter extends SpecificDataStreamAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/opcua"; + + private static final String OPC_SERVER_HOST = "OPC_SERVER_HOST"; + private static final String OPC_SERVER_PORT = "OPC_SERVER_PORT"; + private static final String NAMESPACE_INDEX = "NAMESPACE_INDEX"; + private static final String NODE_ID = "NODE_ID"; + + private String opcUaServer; + private String namespaceIndex; + private String nodeId; + private String port; + + private Map event; + + private OpcUa opcUa; + + private int numberProperties; + + + public OpcUaAdapter() { + this.event = new HashMap<>(); + this.numberProperties = 0; + } + + public OpcUaAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + + getConfigurations(adapterDescription); + + this.event = new HashMap<>(); + this.numberProperties = 0; + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + + SpecificAdapterStreamDescription description = SpecificDataStreamAdapterBuilder.create(ID, "OPC UA", "Read values form an opc ua server") + .iconUrl("opc.jpg") + .category(AdapterType.Generic, AdapterType.Manufacturing) + .requiredTextParameter(Labels.from(OPC_SERVER_HOST, "OPC Server", "Example: test-server.com (No leading opc.tcp://) ")) + .requiredTextParameter(Labels.from(OPC_SERVER_PORT, "OPC Server Port", "Example: 4840")) + .requiredTextParameter(Labels.from(NAMESPACE_INDEX, "Namespace Index", "Example: 2")) + .requiredTextParameter(Labels.from(NODE_ID, "Node Id", "Id of the Node to read the values from")) + .build(); + description.setAppId(ID); + + + return description; + } + + public void onSubscriptionValue(UaMonitoredItem item, DataValue value) { + + String[] keys = item.getReadValueId().getNodeId().getIdentifier().toString().split("\\."); + String key; + + if (keys.length > 0) { + key = keys[keys.length - 1]; + } else { + key = item.getReadValueId().getNodeId().getIdentifier().toString(); + } + event.put(key, value.getValue().getValue()); + + if (event.keySet().size() == this.numberProperties) { + adapterPipeline.process(event); + System.out.println(event); + } + } + + + @Override + public void startAdapter() throws AdapterException { + this.opcUa = new OpcUa(opcUaServer, Integer.parseInt(port), Integer.parseInt(namespaceIndex), nodeId); + try { + this.opcUa.connect(); + + List allNodes = this.opcUa.browseNode(); + List nodeIds = new ArrayList<>(); + + + for (ReferenceDescription rd : allNodes) { + rd.getNodeId().local().ifPresent(nodeId -> nodeIds.add(nodeId)); + } + + this.numberProperties = nodeIds.size(); + this.opcUa.createListSubscription(nodeIds, this); + } catch (Exception e) { + throw new AdapterException("Could not connect to OPC-UA server! Server: " + opcUaServer + " Port: " + port + + " NamespaceIndex: " + namespaceIndex + " NodeId: " + nodeId); + } + } + + @Override + public void stopAdapter() throws AdapterException { + // close connection + this.opcUa.disconnect(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new OpcUaAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + + GuessSchema guessSchema = new GuessSchema(); + EventSchema eventSchema = new EventSchema(); + List allProperties = new ArrayList<>(); + + + getConfigurations(adapterDescription); + + OpcUa opc = new OpcUa(opcUaServer, Integer.parseInt(port), Integer.parseInt(namespaceIndex), nodeId); + try { + opc.connect(); + List res = opc.browseNode(); + + for (ReferenceDescription r : res) { + allProperties.add(PrimitivePropertyBuilder + .create(OpcUaTypes.getType((UInteger) r.getTypeDefinition().getIdentifier()), r.getBrowseName().getName()) + .build()); + } + + opc.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + eventSchema.setEventProperties(allProperties); + guessSchema.setEventSchema(eventSchema); + + return guessSchema; + } + + @Override + public String getId() { + return ID; + } + + private void getConfigurations(SpecificAdapterStreamDescription adapterDescription) { + List all = adapterDescription.getConfig(); + + for (StaticProperty sp : all) { + if (sp.getInternalName().equals(OPC_SERVER_HOST)) { + this.opcUaServer = ((FreeTextStaticProperty) sp).getValue(); + } else if (sp.getInternalName().equals(OPC_SERVER_PORT)) { + this.port = ((FreeTextStaticProperty) sp).getValue(); + } else if (sp.getInternalName().equals(NAMESPACE_INDEX)) { + this.namespaceIndex = ((FreeTextStaticProperty) sp).getValue(); + }else { + this.nodeId = ((FreeTextStaticProperty) sp).getValue(); + } + + } + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTest.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTest.java new file mode 100644 index 0000000000..b80040056f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTest.java @@ -0,0 +1,334 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific.opcua; + + +import com.github.jsonldjava.shaded.com.google.common.collect.Lists; +import org.eclipse.milo.opcua.sdk.client.OpcUaClient; +import org.eclipse.milo.opcua.sdk.client.api.config.OpcUaClientConfig; +import org.eclipse.milo.opcua.sdk.client.api.subscriptions.UaMonitoredItem; +import org.eclipse.milo.opcua.sdk.client.api.subscriptions.UaSubscription; +import org.eclipse.milo.opcua.stack.client.UaTcpStackClient; +import org.eclipse.milo.opcua.stack.core.AttributeId; +import org.eclipse.milo.opcua.stack.core.Identifiers; +import org.eclipse.milo.opcua.stack.core.security.SecurityPolicy; +import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; +import org.eclipse.milo.opcua.stack.core.types.builtin.LocalizedText; +import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; +import org.eclipse.milo.opcua.stack.core.types.builtin.QualifiedName; +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.eclipse.milo.opcua.stack.core.types.enumerated.*; +import org.eclipse.milo.opcua.stack.core.types.structured.*; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; + +import static org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.Unsigned.uint; +import static org.eclipse.milo.opcua.stack.core.util.ConversionUtil.toList; + +public class OpcUaTest { + + // private OpcUaClient myClient; + private static String opcServerURL = "opc.tcp://141.21.43.39:4840"; + // private static String opcServerURL = "opc.tcp://192.168.0.144:4840"; + private static final AtomicLong clientHandles = new AtomicLong(1L); + + + public static void main(String... args) throws Exception { + + OpcUaClient client = init(); +// client.connect().get(); + client.connect().get(); + + NodeId node1 = new NodeId(4, "|var|CODESYS Control for Raspberry Pi SL.Application.PLC_PRG.auto_gruen"); + NodeId node2 = new NodeId(4, "|var|CODESYS Control for Raspberry Pi SL.Application.PLC_PRG.auto_rot"); + NodeId node3 = new NodeId(4, "|var|CODESYS Control for Raspberry Pi SL.Application.PLC_PRG.fuss_rot"); + NodeId node4 = new NodeId(4, "|var|CODESYS Control for Raspberry Pi SL.Application.PLC_PRG"); + +// browseNodeTest("", client, node4); + +// CompletableFuture va1 = client.readValue(0, TimestampsToReturn.Both, node1); +// CompletableFuture va2 = client.readValue(0, TimestampsToReturn.Both, node2); +// CompletableFuture va3 = client.readValue(0, TimestampsToReturn.Both, node3); +// +// +// System.out.println("Auto grün: " + va1.get().getValue()); +// System.out.println("Auto rot: " + va2.get().getValue()); +// System.out.println("Fußgänger rot: " + va3.get().getValue()); + + /* JSONParser parser = new JSONParser(); + JSONObject json = (JSONObject) parser.parse(exchange.getIn().getBody().toString());*/ + + createListSubscription(client, Arrays.asList(node1, node2)); +// createSubscription(client, node1); +// createSubscription(client, node2); + + // let the example run for 10 seconds then terminate + Thread.sleep(100000000); + +// client.disconnect(); + + } + + + + private static OpcUaClient init() throws Exception{ + EndpointDescription[] endpoints = UaTcpStackClient.getEndpoints(opcServerURL).get(); + + EndpointDescription tmpEndpoint = endpoints[0]; + tmpEndpoint = updateEndpointUrl(tmpEndpoint, "141.21.43.39"); + endpoints = new EndpointDescription[]{tmpEndpoint}; + + EndpointDescription endpoint = Arrays.stream(endpoints) + .filter(e -> e.getSecurityPolicyUri().equals(SecurityPolicy.None.getSecurityPolicyUri())) + .findFirst().orElseThrow(() -> new Exception("no desired endpoints returned")); + + OpcUaClientConfig config = OpcUaClientConfig.builder() + .setApplicationName(LocalizedText.english("eclipse milo opc-ua client")) + .setApplicationUri("urn:eclipse:milo:examples:client") + .setEndpoint(endpoint) + .build(); + + return new OpcUaClient(config); + } + + private static EndpointDescription updateEndpointUrl( + EndpointDescription original, String hostname) throws URISyntaxException { + + URI uri = new URI(original.getEndpointUrl()).parseServerAuthority(); + + String endpointUrl = String.format( + "%s://%s:%s%s", + uri.getScheme(), + hostname, + uri.getPort(), + uri.getPath() + ); + + return new EndpointDescription( + endpointUrl, + original.getServer(), + original.getServerCertificate(), + original.getSecurityMode(), + original.getSecurityPolicyUri(), + original.getUserIdentityTokens(), + original.getTransportProfileUri(), + original.getSecurityLevel() + ); + } + + + private static void onSubscriptionValue(UaMonitoredItem item, DataValue value) { + System.out.println( + "subscription value received: " + item.getReadValueId().toString() + " " + value.getValue().toString()); + + } + + private static void createListSubscription(OpcUaClient client, List nodes) throws Exception { + /* + * create a subscription @ 1000ms + */ + UaSubscription subscription = client.getSubscriptionManager().createSubscription(1000.0).get(); + + + List> values = new ArrayList<>(); + + for (NodeId node : nodes) { + values.add(client.readValue(0, TimestampsToReturn.Both, node)); + } + + for (CompletableFuture value : values) { + if (value.get().getValue().toString().contains("null")) { + System.out.println("Node has no value"); + } + } + + + List readValues = new ArrayList<>(); + // Read a specific value attribute + for (NodeId node : nodes) { + readValues.add(new ReadValueId(node, AttributeId.Value.uid(), null, QualifiedName.NULL_VALUE)); + } + + List requests = new ArrayList<>(); + + for (ReadValueId readValue : readValues) { + // important: client handle must be unique per item + UInteger clientHandle = uint(clientHandles.getAndIncrement()); + + MonitoringParameters parameters = new MonitoringParameters( + clientHandle, + 1000.0, // sampling interval + null, // filter, null means use default + uint(10), // queue size + true // discard oldest + ); + + requests.add(new MonitoredItemCreateRequest(readValue, MonitoringMode.Reporting, parameters)); + } + + BiConsumer onItemCreated = + (item, id) -> { + item.setValueConsumer(OpcUaTest::onSubscriptionValue); + }; + + List items = subscription.createMonitoredItems( + TimestampsToReturn.Both, + requests, + onItemCreated + ).get(); + + for (UaMonitoredItem item : items) { + NodeId tagId = item.getReadValueId().getNodeId(); + if (item.getStatusCode().isGood()) { + System.out.println("item created for nodeId="+ tagId); + } else { + System.out.println("failed to create item for " + item.getReadValueId().getNodeId() + item.getStatusCode()); + } + } + + } + + + /** + * creates a subcription for the given node + * + * @param client + * @param node + * @throws Exception + */ + private static void createSubscription(OpcUaClient client, NodeId node) throws Exception { + /* + * create a subscription @ 1000ms + */ + UaSubscription subscription = client.getSubscriptionManager().createSubscription(1000.0).get(); + + CompletableFuture value = client.readValue(0, TimestampsToReturn.Both, node); + + if (value.get().getValue().toString().contains("null")) { + System.out.println("Node has no value"); + } else { + // Read a specific value attribute + ReadValueId readValue = new ReadValueId(node, AttributeId.Value.uid(), null, QualifiedName.NULL_VALUE); + + // important: client handle must be unique per item + UInteger clientHandle = uint(clientHandles.getAndIncrement()); + + MonitoringParameters parameters = new MonitoringParameters( + clientHandle, + 1000.0, // sampling interval + null, // filter, null means use default + uint(10), // queue size + true // discard oldest + ); + + MonitoredItemCreateRequest request = new MonitoredItemCreateRequest(readValue, MonitoringMode.Reporting, parameters); + + + BiConsumer onItemCreated = + (item, id) -> { + System.out.println(id); + item.setValueConsumer(OpcUaTest::onSubscriptionValue); + }; + + + List items = subscription.createMonitoredItems( + TimestampsToReturn.Both, + Lists.newArrayList(request), + onItemCreated + ).get(); + + for (UaMonitoredItem item : items) { + NodeId tagId = item.getReadValueId().getNodeId(); + if (item.getStatusCode().isGood()) { + System.out.println("item created for nodeId="+ tagId); + } else { + System.out.println("failed to create item for " + item.getReadValueId().getNodeId() + item.getStatusCode()); + } + } + + } + } + + private static void browseNodeTest(String indent, OpcUaClient client, NodeId browseRoot) { + BrowseDescription browse = new BrowseDescription( + browseRoot, + BrowseDirection.Forward, + Identifiers.References, + true, + uint(NodeClass.Object.getValue() | NodeClass.Variable.getValue()), + uint(BrowseResultMask.All.getValue()) + ); + + try { + BrowseResult browseResult = client.browse(browse).get(); + + List references = toList(browseResult.getReferences()); + + for (ReferenceDescription rd : references) { + System.out.println("====================================================================="); + System.out.println(rd.toString()); + System.out.println(rd.getNodeClass()); + System.out.println("Node={} " + indent + " " + rd.getBrowseName().getName()); + System.out.println("====================================================================="); + // recursively browse to children + rd.getNodeId().local().ifPresent(nodeId -> browseNodeTest(indent + " ", client, nodeId)); + } + } catch (InterruptedException | ExecutionException e) { + System.out.println("Browsing nodeId=" + browseRoot + " failed: " + e.getMessage()); + } + } + + + private List browseNode(String indent, OpcUaClient client, NodeId browseRoot) { + List result = new ArrayList<>(); + + BrowseDescription browse = new BrowseDescription( + browseRoot, + BrowseDirection.Forward, + Identifiers.References, + true, + uint(NodeClass.Object.getValue() | NodeClass.Variable.getValue()), + uint(BrowseResultMask.All.getValue()) + ); + + try { + BrowseResult browseResult = client.browse(browse).get(); + + List references = toList(browseResult.getReferences()); + + for (ReferenceDescription rd : references) { + result.add(rd); + rd.getNodeId().local().ifPresent(nodeId -> browseNode(indent + " ", client, nodeId)); + } + } catch (InterruptedException | ExecutionException e) { + System.out.println("Browsing nodeId=" + browseRoot + " failed: " + e.getMessage()); + } + + return result; + + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTypes.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTypes.java new file mode 100644 index 0000000000..6dec843e28 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/opcua/OpcUaTypes.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific.opcua; + + +import org.eclipse.milo.opcua.stack.core.types.builtin.unsigned.UInteger; +import org.streampipes.sdk.utils.Datatypes; + +public class OpcUaTypes { + + public static Datatypes getType(UInteger o) { + if (UInteger.valueOf(4).equals(o) | UInteger.valueOf(6).equals(o) | UInteger.valueOf(8).equals(o) | UInteger.valueOf(27).equals(o)) { + return Datatypes.Integer; + } else if (UInteger.valueOf(11).equals(o)) { + return Datatypes.Double; + } else if (UInteger.valueOf(10).equals(o) | UInteger.valueOf(26).equals(o)) { + return Datatypes.Float; + } else if (UInteger.valueOf(1).equals(o)) { + return Datatypes.Boolean; + } else if (UInteger.valueOf(12).equals(o)) { + return Datatypes.String; + } + + return Datatypes.String; + } + +} + diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/ros/RosBridgeAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/ros/RosBridgeAdapter.java index ee0c6f3638..4899b8b212 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/ros/RosBridgeAdapter.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/ros/RosBridgeAdapter.java @@ -31,6 +31,7 @@ import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectParser; import org.streampipes.connect.adapter.specific.SpecificDataStreamAdapter; import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventSchema; @@ -51,10 +52,12 @@ public class RosBridgeAdapter extends SpecificDataStreamAdapter { public static final String ID = "http://streampipes.org/adapter/specific/ros"; private static final String ROS_HOST_KEY = "ROS_HOST_KEY"; + private static final String ROS_PORT_KEY = "ROS_PORT_KEY"; private static final String TOPIC_KEY = "TOPIC_KEY"; private String topic; private String host; + private int port; private Ros ros; @@ -70,11 +73,13 @@ public RosBridgeAdapter(SpecificAdapterStreamDescription adapterDescription) { for (StaticProperty sp : all) { if (sp.getInternalName().equals(ROS_HOST_KEY)) { this.host = ((FreeTextStaticProperty) sp).getValue(); + } else if (sp.getInternalName().equals(ROS_PORT_KEY)) { + port = Integer.parseInt(((FreeTextStaticProperty) sp).getValue()); } else { this.topic = ((FreeTextStaticProperty) sp).getValue(); } } - + this.jsonObjectParser = new JsonObjectParser(); } @@ -82,8 +87,10 @@ public RosBridgeAdapter(SpecificAdapterStreamDescription adapterDescription) { public SpecificAdapterStreamDescription declareModel() { SpecificAdapterStreamDescription description = SpecificDataStreamAdapterBuilder.create(ID, "ROS Bridge", "Connect Robots running on ROS") .iconUrl("ros.png") - .requiredTextParameter(Labels.from(ROS_HOST_KEY, "Ros Bridge", "Hostname of the ROS Bridge")) - .requiredTextParameter(Labels.from(TOPIC_KEY, "Topic", "Name of the topic to be connected of the ROS Bridge")) + .category(AdapterType.Manufacturing) + .requiredTextParameter(Labels.from(ROS_HOST_KEY, "Ros Bridge", "Example: test-server.com (No protocol) ")) + .requiredTextParameter(Labels.from(ROS_PORT_KEY, "Port", "Example: 9090")) + .requiredTextParameter(Labels.from(TOPIC_KEY, "Topic", "Example: /battery (Starts with /) ")) .build(); description.setAppId(ID); @@ -93,7 +100,7 @@ public SpecificAdapterStreamDescription declareModel() { @Override public void startAdapter() throws AdapterException { - this.ros = new Ros(this.host); + this.ros = new Ros(this.host, this.port); this.ros.connect(); String topicType = getMethodType(this.ros, this.topic); @@ -152,17 +159,26 @@ public void stopAdapter() throws AdapterException { public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException { String host = null; String topic = null; + int port = 0; for (StaticProperty sp : adapterDescription.getConfig()) { if (sp.getInternalName().equals(ROS_HOST_KEY)) { host = ((FreeTextStaticProperty) sp).getValue(); - } else { + } else if (sp.getInternalName().equals(ROS_PORT_KEY)) { + port = Integer.parseInt(((FreeTextStaticProperty) sp).getValue()); + } + else { topic = ((FreeTextStaticProperty) sp).getValue(); } } - Ros ros = new Ros(host); - ros.connect(); + Ros ros = new Ros(host, port); + + boolean connect = ros.connect(); + + if (!connect) { + throw new AdapterException("Could not connect to ROS bridge Endpoint: " + host + " with port: " + port); + } String topicType = getMethodType(ros, topic); @@ -178,8 +194,6 @@ public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription } } - System.out.println(getNEvents.getEvents().get(0)); - t.interrupt(); ros.disconnect(); diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java index 3cbcc5f675..f20e7d307e 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java @@ -26,6 +26,7 @@ import org.streampipes.connect.adapter.specific.sensemap.model.Sensor; import org.streampipes.connect.adapter.util.PollingSettings; import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; import org.streampipes.model.connect.guess.GuessSchema; import org.streampipes.model.schema.EventProperty; @@ -81,6 +82,7 @@ public SpecificAdapterStreamDescription declareModel() { SpecificAdapterStreamDescription description = SpecificDataStreamAdapterBuilder.create(ID, "OpenSenseMap", "Environment Sensors") .iconUrl("openSenseMap.png") + .category(AdapterType.Environment, AdapterType.OpenData) .requiredMultiValueSelection(Labels.from("sensors", "Sensors", "Select the " + "sensors that are included in the data stream"), Stream .of(SensorNames.ALL_SENSOR_LABELS) diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackAdapter.java new file mode 100644 index 0000000000..aabe2e1203 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackAdapter.java @@ -0,0 +1,104 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.slack; + +import static org.streampipes.sdk.helpers.EpProperties.stringEp; +import static org.streampipes.sdk.helpers.EpProperties.timestampProperty; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.connect.adapter.specific.SpecificDataStreamAdapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.builder.adapter.GuessSchemaBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.vocabulary.SO; + +public class SlackAdapter extends SpecificDataStreamAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/slack"; + + private static final String SlackToken = "slack-token"; + private static final String Timestamp = "timestamp"; + private static final String Message = "message"; + private static final String Author = "author"; + private static final String Channel = "channel"; + + private String slackApiToken; + private Thread thread; + private SlackConsumer consumer; + + public SlackAdapter() { + super(); + } + + public SlackAdapter(SpecificAdapterStreamDescription adapterStreamDescription) { + super(adapterStreamDescription); + ParameterExtractor extractor = new ParameterExtractor(adapterStreamDescription.getConfig()); + this.slackApiToken = extractor.singleValue(SlackToken); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "Slack", "Subscribes to a Slack channel") + .category(AdapterType.SocialMedia) + .iconUrl("slack.png") + .requiredTextParameter(Labels.from(SlackToken, "Slack API Token", "The API token of " + + "your Slack workspace")) + .build(); + } + + @Override + public void startAdapter() throws AdapterException { + this.consumer = new SlackConsumer(adapterPipeline, slackApiToken); + this.thread = new Thread(consumer); + this.thread.start(); + } + + @Override + public void stopAdapter() throws AdapterException { + this.consumer.stop(); + this.thread.stop(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new SlackAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + return GuessSchemaBuilder.create() + .property(timestampProperty(Timestamp)) + .property(stringEp(Labels.from(Author, "Author", "The username of the sender of the " + + "Slack message"), + Author, SO.Text)) + .property(stringEp(Labels.from(Channel, "Channel", "The Slack channel"), Channel, + SO.Text)) + .property(stringEp(Labels.from(Message, "Message", "The Slack message"), + Message, SO.Text)) + .build(); + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackConsumer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackConsumer.java new file mode 100644 index 0000000000..773898284b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/slack/SlackConsumer.java @@ -0,0 +1,75 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.slack; + +import com.ullink.slack.simpleslackapi.SlackSession; +import com.ullink.slack.simpleslackapi.SlackUser; +import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory; +import com.ullink.slack.simpleslackapi.listeners.SlackMessagePostedListener; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class SlackConsumer implements Runnable { + + private AdapterPipeline adapterPipeline; + private String apiToken; + private SlackSession session; + + public SlackConsumer(AdapterPipeline adapterPipeline, String slackApiToken) { + this.adapterPipeline = adapterPipeline; + this.apiToken = slackApiToken; + } + + public void run() { + SlackMessagePostedListener messagePostedListener = (event, session) -> { + String botName = session.sessionPersona().getUserName(); + String channelOnWhichMessageWasPosted = event.getChannel().getName(); + String messageContent = event.getMessageContent(); + SlackUser messageSender = event.getSender(); + + if (!messageSender.getUserName().equals(botName)) { + Map outEvent = new HashMap<>(); + outEvent.put("timestamp", System.currentTimeMillis()); + outEvent.put("channel", channelOnWhichMessageWasPosted); + outEvent.put("author", messageSender.getUserName()); + outEvent.put("message", messageContent); + + adapterPipeline.process(outEvent); + } + }; + + this.session = SlackSessionFactory.createWebSocketSlackSession(apiToken); + try { + this.session.connect(); + } catch (IOException e) { + e.printStackTrace(); + } + this.session.addMessagePostedListener(messagePostedListener); + } + + public void stop() { + try { + this.session.disconnect(); + } catch (IOException e) { + e.printStackTrace(); + } + } +} + + diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaAdapter.java new file mode 100644 index 0000000000..65e25f12ec --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaAdapter.java @@ -0,0 +1,142 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.wikipedia; + +import static org.streampipes.sdk.helpers.EpProperties.*; + +import com.google.gson.Gson; +import org.streampipes.connect.adapter.specific.SpecificDataStreamAdapter; +import org.streampipes.connect.adapter.specific.wikipedia.model.WikipediaModel; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.exception.ParseException; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.builder.adapter.GuessSchemaBuilder; +import org.streampipes.sdk.helpers.Labels; + +public abstract class WikipediaAdapter extends SpecificDataStreamAdapter { + + public static final String TIMESTAMP = "timestamp"; + public static final String TYPE = "type"; + public static final String EVENT_ID = "id"; + public static final String NAMESPACE = "namespace"; + public static final String TITLE = "title"; + public static final String USER = "user"; + public static final String BOT = "bot"; + public static final String MINOR = "minor"; + public static final String OLDLENGTH = "oldlength"; + public static final String NEWLENGTH = "newlength"; + public static final String OLDREVISION = "oldrevision"; + public static final String NEWREVISION = "newrevision"; + public static final String SERVERURL = "serverurl"; + public static final String SERVERNAME = "servername"; + public static final String WIKI = "wiki"; + public static final String URI = "uri"; + public static final String COMMENT = "comment"; + public static final String DOMAIN = "domain"; + + private static final String VocabPrefix = "http://wikipedia.org/"; + private static final String WikipediaApiUrl = "https://stream.wikimedia" + + ".org/v2/stream/recentchange"; + + private Thread thread; + private String type; + private WikipediaSseConsumer consumer; + + public WikipediaAdapter(SpecificAdapterStreamDescription adapterStreamDescription, String type) { + super(adapterStreamDescription); + this.type = type; + } + + public WikipediaAdapter() { + super(); + } + + @Override + public void startAdapter() throws AdapterException { + Gson gson = new Gson(); + Runnable runnable = () -> { + this.consumer = new WikipediaSseConsumer(); + try { + this.consumer.consumeEventStream(WikipediaApiUrl, + event -> { + WikipediaModel wikipediaModel = gson.fromJson(event, WikipediaModel.class); + if (wikipediaModel != null && wikipediaModel.getType() != null) { + if (wikipediaModel.getType().equals(type)) { + adapterPipeline.process(new WikipediaModelConverter(wikipediaModel).makeMap()); + } + } + }); + } catch (Exception e) { + e.printStackTrace(); + } + }; + + this.thread = new Thread(runnable); + this.thread.start(); + } + + @Override + public void stopAdapter() throws AdapterException { + if (this.thread != null) { + this.consumer.stop(); + } + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) throws AdapterException, ParseException { + return GuessSchemaBuilder.create() + .property(timestampProperty(TIMESTAMP)) + .property(stringEp(Labels.from(EVENT_ID, "ID", ""), EVENT_ID, dp(EVENT_ID))) + .property(doubleEp(Labels.from(TYPE, "Type", "The change type (edit|new)"), + TYPE, dp(TYPE))) + .property(integerEp(Labels.from(NAMESPACE, "Namespace", + "The Wikipedia namespace"), NAMESPACE, dp(NAMESPACE))) + .property(stringEp(Labels.from(TITLE, "Title", "The article title"), + TITLE, dp(TITLE))) + .property(stringEp(Labels.from(USER, "User", "The user ID"), + USER, dp(USER))) + .property(booleanEp(Labels.from(BOT, "Bot", "Edited by a bot"), + BOT, dp(BOT))) + .property(booleanEp(Labels.from(MINOR, "Minor", "Minor edit"), + MINOR, dp(MINOR))) + .property(integerEp(Labels.from(OLDLENGTH, "Old length", ""), + OLDLENGTH, dp(OLDLENGTH))) + .property(integerEp(Labels.from(NEWLENGTH, "New length", ""), + NEWLENGTH, dp(NEWLENGTH))) + .property(longEp(Labels.from(OLDREVISION, "Old revision ID", ""), + OLDREVISION, dp(OLDREVISION))) + .property(longEp(Labels.from(NEWREVISION, "New revision ID", ""), + NEWREVISION, dp(NEWREVISION))) + .property(stringEp(Labels.from(SERVERURL, "Server URL", ""), + SERVERURL, dp(SERVERURL))) + .property(stringEp(Labels.from(SERVERNAME, "Server Name", ""), + SERVERNAME, dp(SERVERNAME))) + .property(stringEp(Labels.from(WIKI, "Wiki Name", ""), + WIKI, dp(WIKI))) + .property(stringEp(Labels.from(URI, "Internal URI", ""), + URI, dp(URI))) + .property(stringEp(Labels.from(COMMENT, "Comment", "Comment field"), + COMMENT, dp(COMMENT))) + .property(stringEp(Labels.from(DOMAIN, "Domain", "Wiki Domain"), + DOMAIN, dp(DOMAIN))) + .build(); + } + + public String dp(String domainPropertyName) { + return VocabPrefix + domainPropertyName; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaEditedArticlesAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaEditedArticlesAdapter.java new file mode 100644 index 0000000000..b840fa28d7 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaEditedArticlesAdapter.java @@ -0,0 +1,55 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.wikipedia; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; + +public class WikipediaEditedArticlesAdapter extends WikipediaAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/wikipedia/edit"; + + private static final String Type = "edit"; + + public WikipediaEditedArticlesAdapter(SpecificAdapterStreamDescription adapterStreamDescription) { + super(adapterStreamDescription, Type); + } + + public WikipediaEditedArticlesAdapter() { + super(); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "Wikipedia Edits", "Continuously publishes" + + " recent Wikipedia edits") + .category(AdapterType.SocialMedia, AdapterType.OpenData) + .iconUrl("wikipedia.png") + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new WikipediaEditedArticlesAdapter(adapterDescription); + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaModelConverter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaModelConverter.java new file mode 100644 index 0000000000..2ea40eac1f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaModelConverter.java @@ -0,0 +1,54 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.wikipedia; + +import static org.streampipes.connect.adapter.specific.wikipedia.WikipediaAdapter.*; + +import org.streampipes.connect.adapter.specific.wikipedia.model.WikipediaModel; + +import java.util.HashMap; +import java.util.Map; + +public class WikipediaModelConverter { + + private WikipediaModel wikipediaModel; + + public WikipediaModelConverter(WikipediaModel wikipediaModel) { + this.wikipediaModel = wikipediaModel; + } + + public Map makeMap() { + Map event = new HashMap<>(); + event.put(TIMESTAMP, wikipediaModel.getTimestamp()); + event.put(TYPE, wikipediaModel.getType()); + event.put(EVENT_ID, wikipediaModel.getId()); + event.put(NAMESPACE, wikipediaModel.getNamespace()); + event.put(TITLE, wikipediaModel.getTitle()); + event.put(USER, wikipediaModel.getUser()); + event.put(BOT, wikipediaModel.getBot()); + event.put(MINOR, wikipediaModel.getMinor()); + event.put(OLDLENGTH, wikipediaModel.getLength().getOld()); + event.put(NEWLENGTH, wikipediaModel.getLength().getNew()); + event.put(OLDREVISION, wikipediaModel.getRevision().getOld()); + event.put(NEWREVISION, wikipediaModel.getRevision().getNew()); + event.put(SERVERURL, wikipediaModel.getServerUrl()); + event.put(SERVERNAME, wikipediaModel.getServerName()); + event.put(WIKI, wikipediaModel.getWiki()); + event.put(URI, wikipediaModel.getMeta().getUri()); + + return event; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaNewArticlesAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaNewArticlesAdapter.java new file mode 100644 index 0000000000..b09ae806aa --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaNewArticlesAdapter.java @@ -0,0 +1,57 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.wikipedia; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.model.AdapterType; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; + +public class WikipediaNewArticlesAdapter extends WikipediaAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/wikipedia/new"; + + private static final String Type = "new"; + + public WikipediaNewArticlesAdapter(SpecificAdapterStreamDescription adapterStreamDescription) { + super(adapterStreamDescription, Type); + } + + public WikipediaNewArticlesAdapter() { + super(); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "Wikipedia New Articles", "Continuously " + + "publishes" + + " articles recently created on Wikipedia") + .category(AdapterType.SocialMedia, AdapterType.OpenData) + .iconUrl("wikipedia.png") + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new WikipediaNewArticlesAdapter(adapterDescription); + } + + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaSseConsumer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaSseConsumer.java new file mode 100644 index 0000000000..3aad3c1a34 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/WikipediaSseConsumer.java @@ -0,0 +1,57 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.wikipedia; + +import org.glassfish.jersey.media.sse.EventInput; +import org.glassfish.jersey.media.sse.InboundEvent; +import org.glassfish.jersey.media.sse.SseFeature; +import org.streampipes.connect.adapter.util.AdapterOutputCollector; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; + +public class WikipediaSseConsumer { + + private Boolean running = true; + + public void consumeEventStream(String url, AdapterOutputCollector consumer) throws Exception { + Client client = ClientBuilder.newBuilder().register(new SseFeature()).build(); + WebTarget target = client.target(url); + EventInput e = null; + while (running) { + Thread.sleep(100); + if (e == null || e.isClosed()) { + // (re)connect + e = target.request().get(EventInput.class); + e.setChunkType("text/event-stream"); + } + + final InboundEvent inboundEvent = e.read(); + if (inboundEvent == null) { + break; + } else { + String data = inboundEvent.readData(); + consumer.onEvent(data); + } + } + } + + public void stop() { + this.running = false; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Length.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Length.java new file mode 100644 index 0000000000..cda7271185 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Length.java @@ -0,0 +1,33 @@ + +package org.streampipes.connect.adapter.specific.wikipedia.model; + +import com.google.gson.annotations.SerializedName; + +import javax.annotation.Generated; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Length { + + @SerializedName("new") + private Long mNew; + @SerializedName("old") + private Long mOld; + + public Long getNew() { + return mNew; + } + + public void setNew(Long newLong) { + mNew = newLong; + } + + public Long getOld() { + return mOld; + } + + public void setOld(Long old) { + mOld = old; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Meta.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Meta.java new file mode 100644 index 0000000000..d162f8115d --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Meta.java @@ -0,0 +1,102 @@ + +package org.streampipes.connect.adapter.specific.wikipedia.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.SerializedName; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Meta { + + @SerializedName("domain") + private String mDomain; + @SerializedName("dt") + private String mDt; + @SerializedName("id") + private String mId; + @SerializedName("offset") + private Long mOffset; + @SerializedName("partition") + private Long mPartition; + @SerializedName("request_id") + private String mRequestId; + @SerializedName("schema_uri") + private String mSchemaUri; + @SerializedName("topic") + private String mTopic; + @SerializedName("uri") + private String mUri; + + public String getDomain() { + return mDomain; + } + + public void setDomain(String domain) { + mDomain = domain; + } + + public String getDt() { + return mDt; + } + + public void setDt(String dt) { + mDt = dt; + } + + public String getId() { + return mId; + } + + public void setId(String id) { + mId = id; + } + + public Long getOffset() { + return mOffset; + } + + public void setOffset(Long offset) { + mOffset = offset; + } + + public Long getPartition() { + return mPartition; + } + + public void setPartition(Long partition) { + mPartition = partition; + } + + public String getRequestId() { + return mRequestId; + } + + public void setRequestId(String requestId) { + mRequestId = requestId; + } + + public String getSchemaUri() { + return mSchemaUri; + } + + public void setSchemaUri(String schemaUri) { + mSchemaUri = schemaUri; + } + + public String getTopic() { + return mTopic; + } + + public void setTopic(String topic) { + mTopic = topic; + } + + public String getUri() { + return mUri; + } + + public void setUri(String uri) { + mUri = uri; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Revision.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Revision.java new file mode 100644 index 0000000000..9d41cab1bd --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/Revision.java @@ -0,0 +1,33 @@ + +package org.streampipes.connect.adapter.specific.wikipedia.model; + +import com.google.gson.annotations.SerializedName; + +import javax.annotation.Generated; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Revision { + + @SerializedName("new") + private Long mNew; + @SerializedName("old") + private Long mOld; + + public Long getNew() { + return mNew; + } + + public void setNew(Long newRevision) { + mNew = newRevision; + } + + public Long getOld() { + return mOld; + } + + public void setOld(Long old) { + mOld = old; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/WikipediaModel.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/WikipediaModel.java new file mode 100644 index 0000000000..e479658929 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/wikipedia/model/WikipediaModel.java @@ -0,0 +1,193 @@ + +package org.streampipes.connect.adapter.specific.wikipedia.model; + +import com.google.gson.annotations.SerializedName; + +import javax.annotation.Generated; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class WikipediaModel { + + @SerializedName("bot") + private Boolean mBot; + @SerializedName("comment") + private String mComment; + @SerializedName("id") + private Long mId; + @SerializedName("length") + private Length mLength; + @SerializedName("meta") + private Meta mMeta; + @SerializedName("minor") + private Boolean mMinor; + @SerializedName("namespace") + private Long mNamespace; + @SerializedName("parsedcomment") + private String mParsedcomment; + @SerializedName("patrolled") + private Boolean mPatrolled; + @SerializedName("revision") + private Revision mRevision; + @SerializedName("server_name") + private String mServerName; + @SerializedName("server_script_path") + private String mServerScriptPath; + @SerializedName("server_url") + private String mServerUrl; + @SerializedName("timestamp") + private Long mTimestamp; + @SerializedName("title") + private String mTitle; + @SerializedName("type") + private String mType; + @SerializedName("user") + private String mUser; + @SerializedName("wiki") + private String mWiki; + + public Boolean getBot() { + return mBot; + } + + public void setBot(Boolean bot) { + mBot = bot; + } + + public String getComment() { + return mComment; + } + + public void setComment(String comment) { + mComment = comment; + } + + public Long getId() { + return mId; + } + + public void setId(Long id) { + mId = id; + } + + public Length getLength() { + return mLength; + } + + public void setLength(Length length) { + mLength = length; + } + + public Meta getMeta() { + return mMeta; + } + + public void setMeta(Meta meta) { + mMeta = meta; + } + + public Boolean getMinor() { + return mMinor; + } + + public void setMinor(Boolean minor) { + mMinor = minor; + } + + public Long getNamespace() { + return mNamespace; + } + + public void setNamespace(Long namespace) { + mNamespace = namespace; + } + + public String getParsedcomment() { + return mParsedcomment; + } + + public void setParsedcomment(String parsedcomment) { + mParsedcomment = parsedcomment; + } + + public Boolean getPatrolled() { + return mPatrolled; + } + + public void setPatrolled(Boolean patrolled) { + mPatrolled = patrolled; + } + + public Revision getRevision() { + return mRevision; + } + + public void setRevision(Revision revision) { + mRevision = revision; + } + + public String getServerName() { + return mServerName; + } + + public void setServerName(String serverName) { + mServerName = serverName; + } + + public String getServerScriptPath() { + return mServerScriptPath; + } + + public void setServerScriptPath(String serverScriptPath) { + mServerScriptPath = serverScriptPath; + } + + public String getServerUrl() { + return mServerUrl; + } + + public void setServerUrl(String serverUrl) { + mServerUrl = serverUrl; + } + + public Long getTimestamp() { + return mTimestamp; + } + + public void setTimestamp(Long timestamp) { + mTimestamp = timestamp; + } + + public String getTitle() { + return mTitle; + } + + public void setTitle(String title) { + mTitle = title; + } + + public String getType() { + return mType; + } + + public void setType(String type) { + mType = type; + } + + public String getUser() { + return mUser; + } + + public void setUser(String user) { + mUser = user; + } + + public String getWiki() { + return mWiki; + } + + public void setWiki(String wiki) { + mWiki = wiki; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/util/AdapterOutputCollector.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/util/AdapterOutputCollector.java new file mode 100644 index 0000000000..65d50b9b0f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/util/AdapterOutputCollector.java @@ -0,0 +1,21 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.util; + +public interface AdapterOutputCollector { + + void onEvent(String data); +} diff --git a/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParserTest.java b/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParserTest.java index 1350049b73..1f1a0a2a23 100644 --- a/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParserTest.java +++ b/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParserTest.java @@ -51,39 +51,6 @@ public void parseOneEvent() throws AdapterException { assertEquals(parsedStringEvent, "{\"one\":1}"); } - - @Test - public void parseThreeEvents() throws AdapterException { - - String jo = getJsonArrayWithThreeElements(); - JsonObjectParser parser = new JsonObjectParser(); - - List parsedEvent = parser.parseNEvents(getInputStream(jo), 3); - - assertEquals(1, parsedEvent.size()); - String parsedStringEventOne = new String(parsedEvent.get(0), StandardCharsets.UTF_8); - - assertEquals( "{\"one\":1}", parsedStringEventOne); - } - - private String getJsonArrayWithThreeElements() { - - String jo = new JSONObject() - .put("one", 1) - .toString(); - - jo += "\n" + new JSONObject() - .put("one", 2) - .toString(); - - jo += "\n" + new JSONObject() - .put("one", 3) - .toString(); - - return jo; - } - - @Test public void parseMoreThenExist() throws AdapterException { diff --git a/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/transform/value/TimestampTransformRuleTest.java b/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/transform/value/TimestampTransformRuleTest.java index 2371b3f03b..d07863952a 100644 --- a/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/transform/value/TimestampTransformRuleTest.java +++ b/streampipes-connect/src/test/java/org/streampipes/connect/adapter/generic/transform/value/TimestampTransformRuleTest.java @@ -16,230 +16,223 @@ package org.streampipes.connect.adapter.generic.transform.value; -import org.junit.Test; -import org.streampipes.model.schema.*; - -import java.util.*; - -import static org.junit.Assert.assertEquals; - public class TimestampTransformRuleTest { - @Test - public void transformListFormatString() { - EventSchema eventSchema = new EventSchema(); - EventPropertyList eventPropertyList = new EventPropertyList(); - eventPropertyList.setRuntimeName("list"); - EventProperty eventPropertyValue = new EventPropertyPrimitive(); - eventPropertyValue.setLabel("value"); - eventPropertyValue.setRuntimeName("value"); - eventPropertyList.setEventProperty(eventPropertyValue); - eventSchema.setEventProperties(Collections.singletonList(eventPropertyList)); - - Map event = new HashMap<>(); - Map subEvent = new HashMap<>(); - subEvent.put("value", "2019-03-11T20:50:38.138Z"); - event.put("list",subEvent); - - List keys = new ArrayList<>(); - keys.add("list"); - keys.add("value"); - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", 1000); - - Map result = timestampTranformationRule.transform(event); - - assertEquals(1, result.keySet().size()); - assertEquals( 1552333838138L, ((Map) result.get(eventPropertyList.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); - } - - @Test - public void transformListTimeUnit() { - EventSchema eventSchema = new EventSchema(); - EventPropertyList eventPropertyList = new EventPropertyList(); - eventPropertyList.setRuntimeName("list"); - EventProperty eventPropertyValue = new EventPropertyPrimitive(); - eventPropertyValue.setLabel("value"); - eventPropertyValue.setRuntimeName("value"); - eventPropertyList.setEventProperty(eventPropertyValue); - eventSchema.setEventProperties(Collections.singletonList(eventPropertyList)); - - Map event = new HashMap<>(); - Map subEvent = new HashMap<>(); - subEvent.put("value", 1552380411); - event.put("list",subEvent); - - List keys = new ArrayList<>(); - keys.add("list"); - keys.add("value"); - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", 1000); - - Map result = timestampTranformationRule.transform(event); - - assertEquals(1, result.keySet().size()); - assertEquals( 1552380411000L, ((Map) result.get(eventPropertyList.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); - - } - - - @Test - public void transformNestedFormatString() { - EventSchema eventSchema = new EventSchema(); - EventPropertyNested eventPropertyMainKey = new EventPropertyNested(); - eventPropertyMainKey.setLabel("mainKey"); - eventPropertyMainKey.setRuntimeName("mainKey"); - EventProperty eventPropertyValue = new EventPropertyPrimitive(); - eventPropertyValue.setLabel("value"); - eventPropertyValue.setRuntimeName("value"); - eventPropertyMainKey.setEventProperties(Collections.singletonList(eventPropertyValue)); - eventSchema.setEventProperties(Collections.singletonList(eventPropertyMainKey)); - - Map event = new HashMap<>(); - Map subEvent = new HashMap<>(); - subEvent.put("value", "2009-12-31"); - event.put("mainKey",subEvent); - - List keys = new ArrayList<>(); - keys.add("mainKey"); - keys.add("value"); - - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd", 1000); - - Map result = timestampTranformationRule.transform(event); - - assertEquals(1, result.keySet().size()); - assertEquals(1262214000000L, ((Map) result.get(eventPropertyMainKey.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); - - } - - @Test - public void transformNestedTimeUnit() { - EventSchema eventSchema = new EventSchema(); - EventPropertyNested eventPropertyMainKey = new EventPropertyNested(); - eventPropertyMainKey.setLabel("mainKey"); - eventPropertyMainKey.setRuntimeName("mainKey"); - EventProperty eventPropertyValue = new EventPropertyPrimitive(); - eventPropertyValue.setLabel("value"); - eventPropertyValue.setRuntimeName("value"); - eventPropertyMainKey.setEventProperties(Collections.singletonList(eventPropertyValue)); - eventSchema.setEventProperties(Collections.singletonList(eventPropertyMainKey)); - - Map event = new HashMap<>(); - Map subEvent = new HashMap<>(); - subEvent.put("value", 1262214000); - event.put("mainKey",subEvent); - - List keys = new ArrayList<>(); - keys.add("mainKey"); - keys.add("value"); - - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd", 1000); - - Map result = timestampTranformationRule.transform(event); - - assertEquals(1, result.keySet().size()); - assertEquals(1262214000000L, ((Map) result.get(eventPropertyMainKey.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); - - } - - - @Test - public void transformMultiEventFormatString() { - EventSchema eventSchema = new EventSchema(); - EventProperty eventPropertyValue1 = new EventPropertyPrimitive(); - eventPropertyValue1.setLabel("value1"); - eventPropertyValue1.setRuntimeName("value1"); - EventProperty eventPropertyValue2 = new EventPropertyPrimitive(); - eventPropertyValue2.setLabel("value2"); - eventPropertyValue2.setRuntimeName("value2"); - eventSchema.addEventProperty(eventPropertyValue1); - eventSchema.addEventProperty(eventPropertyValue2); - - List keys = new ArrayList<>(); - keys.add("value2"); - - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd HH:mm:ss", 1000); - - Map event = new HashMap<>(); - event.put("value1", 0.0); - event.put("value2", "2019-03-15 10:00:00"); - - Map result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1552640400000L, result.get(eventPropertyValue2.getLabel())); - - - event = new HashMap<>(); - event.put("value1", 20.0); - event.put("value2", "2019-03-15 15:23:00"); - - result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1552659780000L, result.get(eventPropertyValue2.getRuntimeName())); - - - event = new HashMap<>(); - event.put("value1", 0.0); - event.put("value2", "2027-012-15 21:53:50"); - - result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1828904030000L, result.get(eventPropertyValue2.getRuntimeName())); - } - - @Test - public void transformMultiEventTimeUnit() { - EventSchema eventSchema = new EventSchema(); - EventProperty eventPropertyValue1 = new EventPropertyPrimitive(); - eventPropertyValue1.setLabel("value1"); - eventPropertyValue1.setRuntimeName("value1"); - EventProperty eventPropertyValue2 = new EventPropertyPrimitive(); - eventPropertyValue2.setLabel("value2"); - eventPropertyValue2.setRuntimeName("value2"); - eventSchema.addEventProperty(eventPropertyValue1); - eventSchema.addEventProperty(eventPropertyValue2); - - List keys = new ArrayList<>(); - keys.add("value2"); - - - TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, - TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd HH:mm:ss", 1000); - - Map event = new HashMap<>(); - event.put("value1", 0.0); - event.put("value2", 1552640400); - - Map result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1552640400000L, result.get(eventPropertyValue2.getLabel())); - - - event = new HashMap<>(); - event.put("value1", 20.0); - event.put("value2", 1552659780); - - result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1552659780000L, result.get(eventPropertyValue2.getRuntimeName())); - - - event = new HashMap<>(); - event.put("value1", 0.0); - event.put("value2", 1828904030); - - result = timestampTranformationRule.transform(event); - assertEquals(2, result.keySet().size()); - assertEquals(1828904030000L, result.get(eventPropertyValue2.getRuntimeName())); - } +// @Test +// public void transformListFormatString() { +// EventSchema eventSchema = new EventSchema(); +// EventPropertyList eventPropertyList = new EventPropertyList(); +// eventPropertyList.setRuntimeName("list"); +// EventProperty eventPropertyValue = new EventPropertyPrimitive(); +// eventPropertyValue.setLabel("value"); +// eventPropertyValue.setRuntimeName("value"); +// eventPropertyList.setEventProperty(eventPropertyValue); +// eventSchema.setEventProperties(Collections.singletonList(eventPropertyList)); +// +// Map event = new HashMap<>(); +// Map subEvent = new HashMap<>(); +// subEvent.put("value", "2019-03-11T20:50:38.138Z"); +// event.put("list",subEvent); +// +// List keys = new ArrayList<>(); +// keys.add("list"); +// keys.add("value"); +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", 1000); +// +// Map result = timestampTranformationRule.transform(event); +// +// assertEquals(1, result.keySet().size()); +// assertEquals( 1552333838138L, ((Map) result.get(eventPropertyList.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); +// } + +// @Test +// public void transformListTimeUnit() { +// EventSchema eventSchema = new EventSchema(); +// EventPropertyList eventPropertyList = new EventPropertyList(); +// eventPropertyList.setRuntimeName("list"); +// EventProperty eventPropertyValue = new EventPropertyPrimitive(); +// eventPropertyValue.setLabel("value"); +// eventPropertyValue.setRuntimeName("value"); +// eventPropertyList.setEventProperty(eventPropertyValue); +// eventSchema.setEventProperties(Collections.singletonList(eventPropertyList)); +// +// Map event = new HashMap<>(); +// Map subEvent = new HashMap<>(); +// subEvent.put("value", 1552380411); +// event.put("list",subEvent); +// +// List keys = new ArrayList<>(); +// keys.add("list"); +// keys.add("value"); +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", 1000); +// +// Map result = timestampTranformationRule.transform(event); +// +// assertEquals(1, result.keySet().size()); +// assertEquals( 1552380411000L, ((Map) result.get(eventPropertyList.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); +// +// } +// +// +// @Test +// public void transformNestedFormatString() { +// EventSchema eventSchema = new EventSchema(); +// EventPropertyNested eventPropertyMainKey = new EventPropertyNested(); +// eventPropertyMainKey.setLabel("mainKey"); +// eventPropertyMainKey.setRuntimeName("mainKey"); +// EventProperty eventPropertyValue = new EventPropertyPrimitive(); +// eventPropertyValue.setLabel("value"); +// eventPropertyValue.setRuntimeName("value"); +// eventPropertyMainKey.setEventProperties(Collections.singletonList(eventPropertyValue)); +// eventSchema.setEventProperties(Collections.singletonList(eventPropertyMainKey)); +// +// Map event = new HashMap<>(); +// Map subEvent = new HashMap<>(); +// subEvent.put("value", "2009-12-31"); +// event.put("mainKey",subEvent); +// +// List keys = new ArrayList<>(); +// keys.add("mainKey"); +// keys.add("value"); +// +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd", 1000); +// +// Map result = timestampTranformationRule.transform(event); +// +// assertEquals(1, result.keySet().size()); +// assertEquals(1262214000000L, ((Map) result.get(eventPropertyMainKey.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); +// +// } +// +// @Test +// public void transformNestedTimeUnit() { +// EventSchema eventSchema = new EventSchema(); +// EventPropertyNested eventPropertyMainKey = new EventPropertyNested(); +// eventPropertyMainKey.setLabel("mainKey"); +// eventPropertyMainKey.setRuntimeName("mainKey"); +// EventProperty eventPropertyValue = new EventPropertyPrimitive(); +// eventPropertyValue.setLabel("value"); +// eventPropertyValue.setRuntimeName("value"); +// eventPropertyMainKey.setEventProperties(Collections.singletonList(eventPropertyValue)); +// eventSchema.setEventProperties(Collections.singletonList(eventPropertyMainKey)); +// +// Map event = new HashMap<>(); +// Map subEvent = new HashMap<>(); +// subEvent.put("value", 1262214000); +// event.put("mainKey",subEvent); +// +// List keys = new ArrayList<>(); +// keys.add("mainKey"); +// keys.add("value"); +// +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd", 1000); +// +// Map result = timestampTranformationRule.transform(event); +// +// assertEquals(1, result.keySet().size()); +// assertEquals(1262214000000L, ((Map) result.get(eventPropertyMainKey.getRuntimeName())).get(eventPropertyValue.getRuntimeName())); +// +// } +// +// +// @Test +// public void transformMultiEventFormatString() { +// EventSchema eventSchema = new EventSchema(); +// EventProperty eventPropertyValue1 = new EventPropertyPrimitive(); +// eventPropertyValue1.setLabel("value1"); +// eventPropertyValue1.setRuntimeName("value1"); +// EventProperty eventPropertyValue2 = new EventPropertyPrimitive(); +// eventPropertyValue2.setLabel("value2"); +// eventPropertyValue2.setRuntimeName("value2"); +// eventSchema.addEventProperty(eventPropertyValue1); +// eventSchema.addEventProperty(eventPropertyValue2); +// +// List keys = new ArrayList<>(); +// keys.add("value2"); +// +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.FORMAT_STRING, "yyyy-MM-dd HH:mm:ss", 1000); +// +// Map event = new HashMap<>(); +// event.put("value1", 0.0); +// event.put("value2", "2019-03-15 10:00:00"); +// +// Map result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1552640400000L, result.get(eventPropertyValue2.getLabel())); +// +// +// event = new HashMap<>(); +// event.put("value1", 20.0); +// event.put("value2", "2019-03-15 15:23:00"); +// +// result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1552659780000L, result.get(eventPropertyValue2.getRuntimeName())); +// +// +// event = new HashMap<>(); +// event.put("value1", 0.0); +// event.put("value2", "2027-012-15 21:53:50"); +// +// result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1828904030000L, result.get(eventPropertyValue2.getRuntimeName())); +// } +// +// @Test +// public void transformMultiEventTimeUnit() { +// EventSchema eventSchema = new EventSchema(); +// EventProperty eventPropertyValue1 = new EventPropertyPrimitive(); +// eventPropertyValue1.setLabel("value1"); +// eventPropertyValue1.setRuntimeName("value1"); +// EventProperty eventPropertyValue2 = new EventPropertyPrimitive(); +// eventPropertyValue2.setLabel("value2"); +// eventPropertyValue2.setRuntimeName("value2"); +// eventSchema.addEventProperty(eventPropertyValue1); +// eventSchema.addEventProperty(eventPropertyValue2); +// +// List keys = new ArrayList<>(); +// keys.add("value2"); +// +// +// TimestampTranformationRule timestampTranformationRule = new TimestampTranformationRule(keys, +// TimestampTranformationRuleMode.TIME_UNIT, "yyyy-MM-dd HH:mm:ss", 1000); +// +// Map event = new HashMap<>(); +// event.put("value1", 0.0); +// event.put("value2", 1552640400); +// +// Map result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1552640400000L, result.get(eventPropertyValue2.getLabel())); +// +// +// event = new HashMap<>(); +// event.put("value1", 20.0); +// event.put("value2", 1552659780); +// +// result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1552659780000L, result.get(eventPropertyValue2.getRuntimeName())); +// +// +// event = new HashMap<>(); +// event.put("value1", 0.0); +// event.put("value2", 1828904030); +// +// result = timestampTranformationRule.transform(event); +// assertEquals(2, result.keySet().size()); +// assertEquals(1828904030000L, result.get(eventPropertyValue2.getRuntimeName())); +// } } diff --git a/streampipes-container-embedded/pom.xml b/streampipes-container-embedded/pom.xml index 4d017ac530..2f9b5011f1 100644 --- a/streampipes-container-embedded/pom.xml +++ b/streampipes-container-embedded/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-container-embedded jar diff --git a/streampipes-container-standalone/pom.xml b/streampipes-container-standalone/pom.xml index 431ae64796..a502c54aef 100644 --- a/streampipes-container-standalone/pom.xml +++ b/streampipes-container-standalone/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-container/pom.xml b/streampipes-container/pom.xml index 3b01c9203e..12c5a7e1fe 100644 --- a/streampipes-container/pom.xml +++ b/streampipes-container/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-container Large scale event processing agents with semantic checking for combinability. diff --git a/streampipes-container/src/main/java/org/streampipes/container/api/Element.java b/streampipes-container/src/main/java/org/streampipes/container/api/Element.java index 0b8f53b978..bf4bbbccad 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/api/Element.java +++ b/streampipes-container/src/main/java/org/streampipes/container/api/Element.java @@ -28,6 +28,7 @@ import org.streampipes.container.declarer.Declarer; import org.streampipes.container.declarer.SemanticEventProducerDeclarer; import org.streampipes.container.init.DeclarersSingleton; +import org.streampipes.container.locales.LabelGenerator; import org.streampipes.container.transform.Transformer; import org.streampipes.empire.core.empire.SupportsRdfId; import org.streampipes.empire.core.empire.annotation.InvalidRdfException; @@ -68,10 +69,16 @@ public String getDescription(@PathParam("id") String elementId) { @Path("{id}/assets") @Produces("application/zip") public Response getAssets(@PathParam("id") String elementId) { - return Response - .ok() - .entity(new AssetZipGenerator(elementId).makeZip()) - .build(); + List includedAssets = getDeclarerById(elementId).declareModel().getIncludedAssets(); + try { + return Response + .ok() + .entity(new AssetZipGenerator(elementId, includedAssets).makeZip()) + .build(); + } catch (IOException e) { + e.printStackTrace(); + return Response.status(500).build(); + } } @GET @@ -150,6 +157,15 @@ protected NamedStreamPipesEntity rewrite(NamedStreamPipesEntity desc, String app desc.setUri(uri); desc.setRdfId(new SupportsRdfId.URIKey(URI.create(uri))); + // TODO remove after full internationalization support has been implemented + if (desc.isIncludesLocales()) { + try { + desc = new LabelGenerator(desc).generateLabels(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (desc instanceof DataSourceDescription) { for (SpDataStream stream : ((DataSourceDescription) desc).getSpDataStreams()) { String baseUri = DeclarersSingleton.getInstance().getBaseUri() @@ -159,6 +175,16 @@ protected NamedStreamPipesEntity rewrite(NamedStreamPipesEntity desc, String app + stream.getUri(); stream.setUri(baseUri); stream.setRdfId(new SupportsRdfId.URIKey(URI.create(baseUri))); + // TODO remove after full internationalization support has been implemented + if (stream.isIncludesLocales()) { + try { + LabelGenerator lg = new LabelGenerator(desc); + stream.setName(lg.getElementTitle()); + stream.setDescription(lg.getElementDescription()); + } catch (IOException e) { + e.printStackTrace(); + } + } } } } diff --git a/streampipes-container/src/main/java/org/streampipes/container/api/SepElement.java b/streampipes-container/src/main/java/org/streampipes/container/api/SepElement.java index 68459eac40..2fcc692abd 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/api/SepElement.java +++ b/streampipes-container/src/main/java/org/streampipes/container/api/SepElement.java @@ -72,10 +72,16 @@ public String getDescription(@PathParam("sourceId") String sourceId, @PathParam( @Produces("application/zip") public javax.ws.rs.core.Response getAssets(@PathParam("sourceId") String sourceId, @PathParam ("streamId") String streamId) { - return javax.ws.rs.core.Response - .ok() - .entity(new AssetZipGenerator(streamId).makeZip()) - .build(); + try { + return javax.ws.rs.core.Response + .ok() + .entity(new AssetZipGenerator(streamId, getDeclarerById(streamId).declareModel() + .getIncludedAssets()).makeZip()) + .build(); + } catch (IOException e) { + e.printStackTrace(); + return javax.ws.rs.core.Response.status(500).build(); + } } private Optional getStreamBySourceId(String sourceId, String streamId) { diff --git a/streampipes-container/src/main/java/org/streampipes/container/assets/AssetZipGenerator.java b/streampipes-container/src/main/java/org/streampipes/container/assets/AssetZipGenerator.java index c7ecca6cf3..fd0d232743 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/assets/AssetZipGenerator.java +++ b/streampipes-container/src/main/java/org/streampipes/container/assets/AssetZipGenerator.java @@ -15,25 +15,57 @@ */ package org.streampipes.container.assets; -import com.google.common.io.Resources; -import org.streampipes.commons.zip.ZipFileGenerator; - -import java.io.File; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; public class AssetZipGenerator { - private String elementAppId; + private List includedAssets; + private String appId; + + public AssetZipGenerator(String appId, List includedAssets) { + this.includedAssets = includedAssets; + this.appId = appId; + } - public AssetZipGenerator(String elementAppId) { - this.elementAppId = elementAppId; + public byte[] makeZip() throws IOException { + return makeZipFromAssets(); } - public byte[] makeZip() { - File assetFolder = getAssetFolder(); - return new ZipFileGenerator(assetFolder).makeZipToBytes(); + private byte[] makeZipFromAssets() throws IOException { + byte[] buffer = new byte[1024]; + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ZipOutputStream out = new ZipOutputStream(outputStream); + + for (String asset : includedAssets) { + ZipEntry ze = new ZipEntry(asset); + out.putNextEntry(ze); + + InputStream in = null; + try { + ClassLoader classLoader = this.getClass().getClassLoader(); + in = classLoader.getResourceAsStream(makePath(asset)); + int len; + while ((len = in.read(buffer)) > 0) { + out.write(buffer, 0, len); + } + } catch(Exception e) { + e.printStackTrace(); + } finally { + in.close(); + } + } + out.closeEntry(); + out.close(); + return outputStream.toByteArray(); } - private File getAssetFolder() { - return new File(Resources.getResource(elementAppId).getFile()); + private String makePath(String assetAppendix) { + return this.appId + "/" + assetAppendix; } } diff --git a/streampipes-container/src/main/java/org/streampipes/container/html/page/WelcomePageGeneratorImpl.java b/streampipes-container/src/main/java/org/streampipes/container/html/page/WelcomePageGeneratorImpl.java index ca989fd6c5..b2b81781a3 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/html/page/WelcomePageGeneratorImpl.java +++ b/streampipes-container/src/main/java/org/streampipes/container/html/page/WelcomePageGeneratorImpl.java @@ -20,8 +20,10 @@ import org.streampipes.container.declarer.*; import org.streampipes.container.html.model.DataSourceDescriptionHtml; import org.streampipes.container.html.model.Description; +import org.streampipes.container.locales.LabelGenerator; import org.streampipes.model.graph.DataSinkDescription; +import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Collection; @@ -52,8 +54,19 @@ public List buildUris() { private Description getDescription(Declarer declarer) { Description desc = new Description(); - desc.setName(declarer.declareModel().getName()); - desc.setDescription(declarer.declareModel().getDescription()); + // TODO remove after full internationalization support has been implemented + if (!declarer.declareModel().isIncludesLocales()) { + desc.setName(declarer.declareModel().getName()); + desc.setDescription(declarer.declareModel().getDescription()); + } else { + LabelGenerator lg = new LabelGenerator(declarer.declareModel()); + try { + desc.setName(lg.getElementTitle()); + desc.setDescription(lg.getElementDescription()); + } catch (IOException e) { + e.printStackTrace(); + } + } desc.setType(getType(declarer)); String uri = baseUri; if (declarer instanceof SemanticEventConsumerDeclarer) { diff --git a/streampipes-container/src/main/java/org/streampipes/container/init/DeclarersSingleton.java b/streampipes-container/src/main/java/org/streampipes/container/init/DeclarersSingleton.java index 406a851e53..16025900fc 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/init/DeclarersSingleton.java +++ b/streampipes-container/src/main/java/org/streampipes/container/init/DeclarersSingleton.java @@ -35,6 +35,10 @@ public class DeclarersSingleton { private static DeclarersSingleton instance; + private static final String Http = "http://"; + private static final String Colon = ":"; + private static final String Slash = "/"; + private Map epaDeclarers; private Map producerDeclarers; private Map consumerDeclarers; @@ -149,11 +153,11 @@ public void setHostName(String host) { } public void setRoute(String route) { - this.route = "/" + route + "/"; + this.route = Slash + route + Slash; } public String getBaseUri() { - return "http://" + hostName + ":" + port + route; + return Http + hostName + Colon + port + route; } private void checkAndStartExecutableStreams(SemanticEventProducerDeclarer sourceDeclarer) { diff --git a/streampipes-container/src/main/java/org/streampipes/container/locales/LabelGenerator.java b/streampipes-container/src/main/java/org/streampipes/container/locales/LabelGenerator.java new file mode 100644 index 0000000000..4b4431cb95 --- /dev/null +++ b/streampipes-container/src/main/java/org/streampipes/container/locales/LabelGenerator.java @@ -0,0 +1,131 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.container.locales; + +import static org.streampipes.container.util.LocalesUtil.makePath; + +import org.streampipes.model.base.ConsumableStreamPipesEntity; +import org.streampipes.model.base.NamedStreamPipesEntity; +import org.streampipes.model.graph.DataProcessorDescription; +import org.streampipes.model.output.AppendOutputStrategy; +import org.streampipes.model.output.FixedOutputStrategy; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class LabelGenerator { + + private static final String Delimiter = "."; + private static final String Title = "title"; + private static final String Description = "description"; + + private NamedStreamPipesEntity desc; + + public LabelGenerator(NamedStreamPipesEntity desc) { + this.desc = desc; + } + + public NamedStreamPipesEntity generateLabels() throws IOException { + if (existsLocalesFile()) { + Properties props = makeProperties(); + desc.setName(getTitle(props, desc.getAppId())); + desc.setDescription(getDescription(props, desc.getAppId())); + + if (isConsumable()) { + ((ConsumableStreamPipesEntity) desc).getStaticProperties().forEach(sp -> { + sp.setLabel(getTitle(props, sp.getInternalName())); + sp.setDescription(getDescription(props, sp.getInternalName())); + }); + } + + if (isDataProcessor()) { + ((DataProcessorDescription) desc).getOutputStrategies().forEach(os -> { + if (os instanceof AppendOutputStrategy) { + ((AppendOutputStrategy) os).getEventProperties().forEach(ep -> { + ep.setLabel(getTitle(props, ep.getRuntimeId())); + ep.setDescription(getDescription(props, ep.getRuntimeId())); + }); + } else if (os instanceof FixedOutputStrategy) { + ((FixedOutputStrategy) os).getEventProperties().forEach(ep -> { + ep.setLabel(getTitle(props, ep.getRuntimeId())); + ep.setDescription(getDescription(props, ep.getRuntimeId())); + }); + } + }); + } + } + + return desc; + } + + private Properties makeProperties() throws IOException { + Properties props = new Properties(); + props.load(loadResource()); + + return props; + } + + public String getElementTitle() throws IOException { + Properties props = makeProperties(); + return getTitle(props, desc.getAppId()); + } + + public String getElementDescription() throws IOException { + Properties props = makeProperties(); + return getDescription(props, desc.getAppId()); + } + + private boolean existsLocalesFile() { + return this.getClass().getClassLoader().getResourceAsStream(makePath(desc, + this.desc.getIncludedLocales().get(0))) != null; + } + + private boolean isConsumable() { + return desc instanceof ConsumableStreamPipesEntity; + } + + private boolean isDataProcessor() { + return desc instanceof DataProcessorDescription; + } + + + private InputStream loadResource() { + if (desc.getIncludedLocales().size() > 0) { + return getResourceFile(desc.getIncludedLocales().get(0)); + } else { + throw new IllegalArgumentException("Could not find any language files"); + } + } + + private String getTitle(Properties props, String id) { + return getValue(props, Title, id); + } + + private String getDescription(Properties props, String id) { + return getValue(props, Description, id); + } + + private String getValue(Properties props, String type, String id) { + return props.getProperty(id + Delimiter + type, ""); + } + + + private InputStream getResourceFile(String filename) { + return this.getClass().getClassLoader().getResourceAsStream(makePath(desc, filename)); + } + +} diff --git a/streampipes-container/src/main/java/org/streampipes/container/util/LocalesUtil.java b/streampipes-container/src/main/java/org/streampipes/container/util/LocalesUtil.java new file mode 100644 index 0000000000..58e057ba4a --- /dev/null +++ b/streampipes-container/src/main/java/org/streampipes/container/util/LocalesUtil.java @@ -0,0 +1,27 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.container.util; + +import org.streampipes.model.base.NamedStreamPipesEntity; + +public class LocalesUtil { + + public static String makePath(NamedStreamPipesEntity desc, String assetAppendix) { + return desc.getAppId() + + "/" + + assetAppendix; + } +} diff --git a/streampipes-container/src/main/java/org/streampipes/container/util/Util.java b/streampipes-container/src/main/java/org/streampipes/container/util/Util.java index c534867f52..fa53c6ac4f 100644 --- a/streampipes-container/src/main/java/org/streampipes/container/util/Util.java +++ b/streampipes-container/src/main/java/org/streampipes/container/util/Util.java @@ -22,8 +22,15 @@ import org.streampipes.model.Response; public class Util { + + private static final String Slash = "/"; + public static String getInstanceId(String url, String type, String elemntId) { - return url.replace(DeclarersSingleton.getInstance().getBaseUri() + type + "/" + elemntId + "/", ""); + return url.replace(DeclarersSingleton.getInstance().getBaseUri() + + type + + Slash + + elemntId + + Slash, ""); } public static String toResponseString(String elementId, boolean success) { diff --git a/streampipes-container/src/test/java/de/fzi/cep/sepa/init/DeclarersSingletonTest.java b/streampipes-container/src/test/java/de/fzi/cep/sepa/init/DeclarersSingletonTest.java deleted file mode 100644 index 015915685c..0000000000 --- a/streampipes-container/src/test/java/de/fzi/cep/sepa/init/DeclarersSingletonTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2018 FZI Forschungszentrum Informatik - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package de.fzi.cep.sepa.init; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.Test; - -import org.streampipes.container.declarer.Declarer; -import org.streampipes.container.declarer.SemanticEventProcessingAgentDeclarer; -import org.streampipes.container.init.DeclarersSingleton; -import org.streampipes.model.Response; -import org.streampipes.model.graph.DataProcessorDescription; -import org.streampipes.model.graph.DataProcessorInvocation; - -public class DeclarersSingletonTest { - @Test - public void getInstanceIsSingletonTest() throws Exception { - DeclarersSingleton ds1 = DeclarersSingleton.getInstance(); - DeclarersSingleton ds2 = DeclarersSingleton.getInstance(); - - assertTrue(ds1 == ds2); - } - - @Test - public void addDeclarersTest() throws Exception { - List declarers = new ArrayList<>(); - declarers.add(getSepaDeclarer()); - - DeclarersSingleton.getInstance().addDeclarers(declarers); - assertEquals(DeclarersSingleton.getInstance().getEpaDeclarers().size(), 1); - } - - private SemanticEventProcessingAgentDeclarer getSepaDeclarer() { - return new SemanticEventProcessingAgentDeclarer() { - @Override - public Response invokeRuntime(DataProcessorInvocation invocationGraph) { - return null; - } - - @Override - public Response detachRuntime(String pipelineId) { - return null; - } - - @Override - public DataProcessorDescription declareModel() { - return null; - } - }; - } -} \ No newline at end of file diff --git a/streampipes-container/src/test/java/org/streampipes/container/api/ElementTest.java b/streampipes-container/src/test/java/org/streampipes/container/api/ElementTest.java deleted file mode 100644 index 37a8cfa679..0000000000 --- a/streampipes-container/src/test/java/org/streampipes/container/api/ElementTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2018 FZI Forschungszentrum Informatik - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.streampipes.container.api; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -import org.junit.Test; -import org.streampipes.container.declarer.Declarer; -import org.streampipes.container.declarer.InvocableDeclarer; -import org.streampipes.model.Response; -import org.streampipes.model.base.NamedStreamPipesEntity; -import org.streampipes.model.graph.DataProcessorDescription; -import org.streampipes.model.graph.DataProcessorInvocation; - -import java.util.HashMap; -import java.util.Map; - -public class ElementTest {; - @Test - public void getByIdTest() { - String id = "sepapathName1"; - Map declarers = new HashMap<>(); - declarers.put(id, getDeclarerImpl(id)); - declarers.put("sepapathName2", getDeclarerImpl("sepapathName2")); - TestElementImpl elem = new TestElementImpl(); - elem.setDeclarers(declarers); - - NamedStreamPipesEntity namedSEPAElement = elem.getById(id); - - assertEquals("sepaname", namedSEPAElement.getName()); - assertEquals("sepadescription", namedSEPAElement.getDescription()); - } - - @Test - public void getByIdIsNullTest() { - TestElementImpl elem = new TestElementImpl(); - elem.setDeclarers(new HashMap<>()); - - NamedStreamPipesEntity actual = elem.getById(""); - assertNull(actual); - } - - @Test - public void toJsonLdNullTest() { - SepElement sep = new SepElement(); - assertEquals("{}", sep.toJsonLd(null)); - } - - - private DeclarerImpl getDeclarerImpl(String id) { - return new DeclarerImpl(id); - } - - - private class DeclarerImpl implements InvocableDeclarer { - private String id; - - public DeclarerImpl(String id) { - this.id = id; - } - - @Override - public Response invokeRuntime(DataProcessorInvocation invocationGraph) { - return null; - } - - @Override - public Response detachRuntime(String pipelineId) { - return null; - } - - @Override - public DataProcessorDescription declareModel() { - return new DataProcessorDescription(id, "sepaname", "sepadescription", "sepaiconUrl"); - } - } - - private class TestElementImpl extends Element { - private Map declarers = new HashMap<>(); - - public Map getDeclarers() { - return declarers; - } - - public void setDeclarers(Map declarers) { - this.declarers = declarers; - } - - @Override - protected Map getElementDeclarers() { - return declarers; - } - } -} diff --git a/streampipes-dataformat-json/pom.xml b/streampipes-dataformat-json/pom.xml index a1085f3da8..8b45810d7e 100644 --- a/streampipes-dataformat-json/pom.xml +++ b/streampipes-dataformat-json/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-dataformat/pom.xml b/streampipes-dataformat/pom.xml index 80ef6b231c..d5689b4e3d 100644 --- a/streampipes-dataformat/pom.xml +++ b/streampipes-dataformat/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-logging/pom.xml b/streampipes-logging/pom.xml index 9cb6e1d4ef..a67a776a2f 100644 --- a/streampipes-logging/pom.xml +++ b/streampipes-logging/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-measurement-units/pom.xml b/streampipes-measurement-units/pom.xml index 99f8311373..bbe99d9804 100644 --- a/streampipes-measurement-units/pom.xml +++ b/streampipes-measurement-units/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-measurement-units diff --git a/streampipes-messaging-jms/pom.xml b/streampipes-messaging-jms/pom.xml index b1dd132096..7cb5b2ed8a 100644 --- a/streampipes-messaging-jms/pom.xml +++ b/streampipes-messaging-jms/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-messaging-kafka/pom.xml b/streampipes-messaging-kafka/pom.xml index c88b8ba2d4..f4ad9b73cc 100644 --- a/streampipes-messaging-kafka/pom.xml +++ b/streampipes-messaging-kafka/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-messaging/pom.xml b/streampipes-messaging/pom.xml index 02bbd5c6d2..0f658ff288 100644 --- a/streampipes-messaging/pom.xml +++ b/streampipes-messaging/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-model-client/pom.xml b/streampipes-model-client/pom.xml index fc3fea7ad9..7bea80401d 100644 --- a/streampipes-model-client/pom.xml +++ b/streampipes-model-client/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-model-client jar diff --git a/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/AssetDashboardConfig.java b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/AssetDashboardConfig.java new file mode 100644 index 0000000000..362bc038c8 --- /dev/null +++ b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/AssetDashboardConfig.java @@ -0,0 +1,100 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.model.client.assetdashboard; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +public class AssetDashboardConfig { + + private @SerializedName("_id") String dashboardId; + private @SerializedName("_rev") String rev; + + private String dashboardName; + private String dashboardDescription; + private ImageInfo imageInfo; + private CanvasAttributes attrs; + private String className; + private List children; + + public AssetDashboardConfig() { + } + + public String getDashboardName() { + return dashboardName; + } + + public void setDashboardName(String dashboardName) { + this.dashboardName = dashboardName; + } + + public String getDashboardDescription() { + return dashboardDescription; + } + + public void setDashboardDescription(String dashboardDescription) { + this.dashboardDescription = dashboardDescription; + } + + public CanvasAttributes getAttrs() { + return attrs; + } + + public void setAttrs(CanvasAttributes attrs) { + this.attrs = attrs; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + public ImageInfo getImageInfo() { + return imageInfo; + } + + public void setImageInfo(ImageInfo imageInfo) { + this.imageInfo = imageInfo; + } + + public String getDashboardId() { + return dashboardId; + } + + public void setDashboardId(String dashboardId) { + this.dashboardId = dashboardId; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } +} diff --git a/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasAttributes.java b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasAttributes.java new file mode 100644 index 0000000000..bf129ab305 --- /dev/null +++ b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasAttributes.java @@ -0,0 +1,169 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.model.client.assetdashboard; + +public class CanvasAttributes { + + private float x; + private float y; + private float width; + private float height; + private float scaleX = 1; + private float scaleY = 1; + private float rotation = 0; + private String fill; + private String text; + private String align; + private String fontSize; + + private String id; + private String visualizationId; + + private String brokerUrl; + private String topic; + private String name; + + public CanvasAttributes() { + } + + public float getX() { + return x; + } + + public void setX(float x) { + this.x = x; + } + + public float getY() { + return y; + } + + public void setY(float y) { + this.y = y; + } + + public float getScaleX() { + return scaleX; + } + + public void setScaleX(float scaleX) { + this.scaleX = scaleX; + } + + public float getScaleY() { + return scaleY; + } + + public void setScaleY(float scaleY) { + this.scaleY = scaleY; + } + + public String getFill() { + return fill; + } + + public void setFill(String fill) { + this.fill = fill; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getVisualizationId() { + return visualizationId; + } + + public void setVisualizationId(String visualizationId) { + this.visualizationId = visualizationId; + } + + public float getWidth() { + return width; + } + + public void setWidth(float width) { + this.width = width; + } + + public float getHeight() { + return height; + } + + public void setHeight(float height) { + this.height = height; + } + + public float getRotation() { + return rotation; + } + + public void setRotation(float rotation) { + this.rotation = rotation; + } + + public String getBrokerUrl() { + return brokerUrl; + } + + public void setBrokerUrl(String brokerUrl) { + this.brokerUrl = brokerUrl; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAlign() { + return align; + } + + public void setAlign(String align) { + this.align = align; + } + + public String getFontSize() { + return fontSize; + } + + public void setFontSize(String fontSize) { + this.fontSize = fontSize; + } +} diff --git a/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasElement.java b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasElement.java new file mode 100644 index 0000000000..1544aa1179 --- /dev/null +++ b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/CanvasElement.java @@ -0,0 +1,52 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.model.client.assetdashboard; + +import java.util.List; + +public class CanvasElement { + + private CanvasAttributes attrs; + private String className; + private List children; + + public CanvasElement() { + } + + public CanvasAttributes getAttrs() { + return attrs; + } + + public void setAttrs(CanvasAttributes attrs) { + this.attrs = attrs; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } +} diff --git a/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/ImageInfo.java b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/ImageInfo.java new file mode 100644 index 0000000000..69ec831ca8 --- /dev/null +++ b/streampipes-model-client/src/main/java/org/streampipes/model/client/assetdashboard/ImageInfo.java @@ -0,0 +1,95 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.model.client.assetdashboard; + +public class ImageInfo { + + private String imageName; + private float x; + private float y; + private float width; + private float height; + private float scaleX = 1; + private float scaleY = 1; + private float rotation = 0; + + public ImageInfo() { + } + + public float getX() { + return x; + } + + public void setX(float x) { + this.x = x; + } + + public float getY() { + return y; + } + + public void setY(float y) { + this.y = y; + } + + public float getWidth() { + return width; + } + + public void setWidth(float width) { + this.width = width; + } + + public float getHeight() { + return height; + } + + public void setHeight(float height) { + this.height = height; + } + + public float getScaleX() { + return scaleX; + } + + public void setScaleX(float scaleX) { + this.scaleX = scaleX; + } + + public float getScaleY() { + return scaleY; + } + + public void setScaleY(float scaleY) { + this.scaleY = scaleY; + } + + public String getImageName() { + return imageName; + } + + public void setImageName(String imageName) { + this.imageName = imageName; + } + + public float getRotation() { + return rotation; + } + + public void setRotation(float rotation) { + this.rotation = rotation; + } +} diff --git a/streampipes-model/pom.xml b/streampipes-model/pom.xml index 3a52c21b76..69f051cdc2 100644 --- a/streampipes-model/pom.xml +++ b/streampipes-model/pom.xml @@ -22,7 +22,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 diff --git a/streampipes-model/src/main/java/org/streampipes/model/AdapterType.java b/streampipes-model/src/main/java/org/streampipes/model/AdapterType.java new file mode 100644 index 0000000000..da4714abe7 --- /dev/null +++ b/streampipes-model/src/main/java/org/streampipes/model/AdapterType.java @@ -0,0 +1,43 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.model; + +public enum AdapterType { + + Generic("Generic Adapters", ""), + Finance("Finance", ""), + Environment("Environmental Data", ""), + News("News", ""), + SocialMedia("Social Media", ""), + OpenData("Open Data", ""), + Manufacturing("Production & Manufacturing", ""); + + private String label; + private String description; + + AdapterType(String label, String description) { + this.label = label; + this.description = description; + } + + public String getLabel() { + return label; + } + + public String getDescription() { + return description; + } +} diff --git a/streampipes-model/src/main/java/org/streampipes/model/DataSinkType.java b/streampipes-model/src/main/java/org/streampipes/model/DataSinkType.java index 8438be3b1f..d193481161 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/DataSinkType.java +++ b/streampipes-model/src/main/java/org/streampipes/model/DataSinkType.java @@ -19,27 +19,27 @@ public enum DataSinkType { - VISUALIZATION_CHART("Charts", ""), - VISUALIZATION_GEO("Geospatial Visualization", ""), - STORAGE("Storage", ""), - FORWARD("Forward", ""), - NOTIFICATION("Notifications", ""), - ACTUATOR("Actuators", ""), - UNCATEGORIZED("Uncategorized", ""); - - private String label; - private String description; - - DataSinkType(String label, String description) { - this.label = label; - this.description = description; - } + VISUALIZATION_CHART("Charts", ""), + VISUALIZATION_GEO("Geospatial Visualization", ""), + STORAGE("Storage", ""), + FORWARD("Forward", ""), + NOTIFICATION("Notifications", ""), + ACTUATOR("Actuators", ""), + UNCATEGORIZED("Uncategorized", ""); - public String getLabel() { - return label; - } + private String label; + private String description; - public String getDescription() { - return description; - } + DataSinkType(String label, String description) { + this.label = label; + this.description = description; + } + + public String getLabel() { + return label; + } + + public String getDescription() { + return description; + } } diff --git a/streampipes-model/src/main/java/org/streampipes/model/SpDataStream.java b/streampipes-model/src/main/java/org/streampipes/model/SpDataStream.java index 6c94684957..a95f947f42 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/SpDataStream.java +++ b/streampipes-model/src/main/java/org/streampipes/model/SpDataStream.java @@ -30,137 +30,163 @@ import org.streampipes.model.util.Cloner; import org.streampipes.vocabulary.StreamPipes; -import javax.persistence.*; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import javax.persistence.CascadeType; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.OneToMany; +import javax.persistence.OneToOne; + @RdfsClass(StreamPipes.DATA_STREAM) @Entity public class SpDataStream extends NamedStreamPipesEntity { - private static final long serialVersionUID = -5732549347563182863L; - - private static final String prefix = "urn:fzi.de:eventstream:"; - - @OneToMany(fetch = FetchType.EAGER, - cascade = {CascadeType.PERSIST, CascadeType.MERGE}) - @RdfProperty(StreamPipes.HAS_EVENT_STREAM_QUALITY_DEFINITION) - protected transient List hasEventStreamQualities; - - @OneToMany(fetch = FetchType.EAGER, - cascade = {CascadeType.PERSIST, CascadeType.MERGE}) - @RdfProperty(StreamPipes.HAS_EVENT_STREAM_QUALITY_REQUIREMENT) - protected transient List requiresEventStreamQualities; - - @OneToOne(fetch = FetchType.EAGER, - cascade = {CascadeType.PERSIST, CascadeType.MERGE}) - @RdfProperty(StreamPipes.HAS_GROUNDING) - protected EventGrounding eventGrounding; - - @OneToOne(fetch = FetchType.EAGER,cascade = {CascadeType.ALL}) - @RdfProperty(StreamPipes.HAS_SCHEMA) - protected EventSchema eventSchema; - - @RdfProperty(StreamPipes.HAS_MEASUREMENT_CAPABILTIY) - @OneToMany(fetch = FetchType.EAGER,cascade = {CascadeType.ALL}) - protected List measurementCapability; - - @RdfProperty(StreamPipes.HAS_MEASUREMENT_OBJECT) - @OneToMany(fetch = FetchType.EAGER,cascade = {CascadeType.ALL}) - protected List measurementObject; - - protected List category; - - public SpDataStream(String uri, String name, String description, String iconUrl, List hasEventStreamQualities, - EventGrounding eventGrounding, - EventSchema eventSchema) { - super(uri, name, description, iconUrl); - this.hasEventStreamQualities = hasEventStreamQualities; - this.eventGrounding = eventGrounding; - this.eventSchema = eventSchema; - } - - public SpDataStream(String uri, String name, String description, EventSchema eventSchema) - { - super(uri, name, description); - this.eventSchema = eventSchema; - } - - public SpDataStream() { - super(prefix +RandomStringUtils.randomAlphabetic(6)); - } - - - public SpDataStream(SpDataStream other) { - super(other); - if (other.getEventGrounding() != null) this.eventGrounding = new EventGrounding(other.getEventGrounding()); - if (other.getEventSchema() != null) this.eventSchema = new EventSchema(other.getEventSchema()); - if (other.getHasEventStreamQualities() != null) this.hasEventStreamQualities = other.getHasEventStreamQualities().stream().map(EventStreamQualityDefinition::new).collect(Collectors.toCollection(ArrayList::new)); - if (other.getRequiresEventStreamQualities() != null) this.requiresEventStreamQualities = other.getRequiresEventStreamQualities().stream().map(EventStreamQualityRequirement::new).collect(Collectors.toCollection(ArrayList::new)); - if (other.getMeasurementCapability() != null) this.measurementCapability = new Cloner().mc(other.getMeasurementCapability()); - if (other.getMeasurementObject() != null) this.measurementObject = new Cloner().mo(other.getMeasurementObject()); - - } - - public List getHasEventStreamQualities() { - return hasEventStreamQualities; - } - - public void setHasEventStreamQualities( - List hasEventStreamQualities) { - this.hasEventStreamQualities = hasEventStreamQualities; - } - - - public List getRequiresEventStreamQualities() { - return requiresEventStreamQualities; - } - - public void setRequiresEventStreamQualities( - List requiresEventStreamQualities) { - this.requiresEventStreamQualities = requiresEventStreamQualities; - } - - public EventSchema getEventSchema() { - return eventSchema; - } - - public void setEventSchema(EventSchema eventSchema) { - this.eventSchema = eventSchema; - } - - public EventGrounding getEventGrounding() { - return eventGrounding; - } - - public void setEventGrounding(EventGrounding eventGrounding) { - this.eventGrounding = eventGrounding; - } - - public List getMeasurementCapability() { - return measurementCapability; - } - - public void setMeasurementCapability( - List measurementCapability) { - this.measurementCapability = measurementCapability; - } - - public List getMeasurementObject() { - return measurementObject; - } - - public void setMeasurementObject(List measurementObject) { - this.measurementObject = measurementObject; - } - - public List getCategory() { - return category; - } - - public void setCategory(List category) { - this.category = category; - } - + private static final long serialVersionUID = -5732549347563182863L; + + private static final String prefix = "urn:fzi.de:eventstream:"; + + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @RdfProperty(StreamPipes.HAS_EVENT_STREAM_QUALITY_DEFINITION) + protected transient List hasEventStreamQualities; + + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @RdfProperty(StreamPipes.HAS_EVENT_STREAM_QUALITY_REQUIREMENT) + protected transient List requiresEventStreamQualities; + + @OneToOne(fetch = FetchType.EAGER, + cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @RdfProperty(StreamPipes.HAS_GROUNDING) + protected EventGrounding eventGrounding; + + @OneToOne(fetch = FetchType.EAGER, cascade = {CascadeType.ALL}) + @RdfProperty(StreamPipes.HAS_SCHEMA) + protected EventSchema eventSchema; + + @RdfProperty(StreamPipes.HAS_MEASUREMENT_CAPABILTIY) + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.ALL}) + protected List measurementCapability; + + @RdfProperty(StreamPipes.HAS_MEASUREMENT_OBJECT) + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.ALL}) + protected List measurementObject; + + @RdfProperty(StreamPipes.INDEX) + private int index; + + protected List category; + + public SpDataStream(String uri, String name, String description, String iconUrl, List hasEventStreamQualities, + EventGrounding eventGrounding, + EventSchema eventSchema) { + super(uri, name, description, iconUrl); + this.hasEventStreamQualities = hasEventStreamQualities; + this.eventGrounding = eventGrounding; + this.eventSchema = eventSchema; + } + + public SpDataStream(String uri, String name, String description, EventSchema eventSchema) { + super(uri, name, description); + this.eventSchema = eventSchema; + } + + public SpDataStream() { + super(prefix + RandomStringUtils.randomAlphabetic(6)); + } + + + public SpDataStream(SpDataStream other) { + super(other); + this.index = other.getIndex(); + if (other.getEventGrounding() != null) { + this.eventGrounding = new EventGrounding(other.getEventGrounding()); + } + if (other.getEventSchema() != null) { + this.eventSchema = new EventSchema(other.getEventSchema()); + } + if (other.getHasEventStreamQualities() != null) { + this.hasEventStreamQualities = other.getHasEventStreamQualities().stream().map(EventStreamQualityDefinition::new).collect(Collectors.toCollection(ArrayList::new)); + } + if (other.getRequiresEventStreamQualities() != null) { + this.requiresEventStreamQualities = other.getRequiresEventStreamQualities().stream().map(EventStreamQualityRequirement::new).collect(Collectors.toCollection(ArrayList::new)); + } + if (other.getMeasurementCapability() != null) { + this.measurementCapability = new Cloner().mc(other.getMeasurementCapability()); + } + if (other.getMeasurementObject() != null) { + this.measurementObject = new Cloner().mo(other.getMeasurementObject()); + } + } + + public List getHasEventStreamQualities() { + return hasEventStreamQualities; + } + + public void setHasEventStreamQualities( + List hasEventStreamQualities) { + this.hasEventStreamQualities = hasEventStreamQualities; + } + + + public List getRequiresEventStreamQualities() { + return requiresEventStreamQualities; + } + + public void setRequiresEventStreamQualities( + List requiresEventStreamQualities) { + this.requiresEventStreamQualities = requiresEventStreamQualities; + } + + public EventSchema getEventSchema() { + return eventSchema; + } + + public void setEventSchema(EventSchema eventSchema) { + this.eventSchema = eventSchema; + } + + public EventGrounding getEventGrounding() { + return eventGrounding; + } + + public void setEventGrounding(EventGrounding eventGrounding) { + this.eventGrounding = eventGrounding; + } + + public List getMeasurementCapability() { + return measurementCapability; + } + + public void setMeasurementCapability( + List measurementCapability) { + this.measurementCapability = measurementCapability; + } + + public List getMeasurementObject() { + return measurementObject; + } + + public void setMeasurementObject(List measurementObject) { + this.measurementObject = measurementObject; + } + + public List getCategory() { + return category; + } + + public void setCategory(List category) { + this.category = category; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } } diff --git a/streampipes-model/src/main/java/org/streampipes/model/base/NamedStreamPipesEntity.java b/streampipes-model/src/main/java/org/streampipes/model/base/NamedStreamPipesEntity.java index 131a24c9de..a1075ffa09 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/base/NamedStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/streampipes/model/base/NamedStreamPipesEntity.java @@ -58,6 +58,19 @@ public abstract class NamedStreamPipesEntity extends AbstractStreamPipesEntity { @RdfProperty(StreamPipes.INCLUDES_ASSETS) private boolean includesAssets; + @RdfProperty(StreamPipes.INCLUDES_LOCALES) + private boolean includesLocales; + + @RdfProperty(StreamPipes.INCLUDED_ASSETS) + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.ALL}) + private List includedAssets; + + @RdfProperty(StreamPipes.INCLUDED_LOCALES) + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.ALL}) + private List includedLocales; + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.ALL}) @RdfProperty(StreamPipes.HAS_APPLICATION_LINK) @@ -70,18 +83,18 @@ public abstract class NamedStreamPipesEntity extends AbstractStreamPipesEntity { public NamedStreamPipesEntity() { super(); this.applicationLinks = new ArrayList<>(); + this.includedAssets = new ArrayList<>(); + this.includedLocales = new ArrayList<>(); } public NamedStreamPipesEntity(String elementId) { - super(); + this(); this.elementId = elementId; - this.applicationLinks = new ArrayList<>(); } public NamedStreamPipesEntity(String elementId, String name, String description, String iconUrl) { this(elementId, name, description); this.iconUrl = iconUrl; - this.applicationLinks = new ArrayList<>(); } public NamedStreamPipesEntity(String elementId, String name, String description) { @@ -90,6 +103,8 @@ public NamedStreamPipesEntity(String elementId, String name, String description) this.name = name; this.description = description; this.applicationLinks = new ArrayList<>(); + this.includedAssets = new ArrayList<>(); + this.includedLocales = new ArrayList<>(); } public NamedStreamPipesEntity(NamedStreamPipesEntity other) { @@ -105,6 +120,13 @@ public NamedStreamPipesEntity(NamedStreamPipesEntity other) { } this.appId = other.getAppId(); this.includesAssets = other.isIncludesAssets(); + this.includesLocales = other.isIncludesLocales(); + if (other.getIncludedAssets() != null) { + this.includedAssets = other.getIncludedAssets(); + } + if (other.getIncludedLocales() != null) { + this.includedLocales = other.getIncludedLocales(); + } } public String getName() { @@ -189,6 +211,30 @@ public void setIncludesAssets(boolean includesAssets) { this.includesAssets = includesAssets; } + public List getIncludedAssets() { + return includedAssets; + } + + public void setIncludedAssets(List includedAssets) { + this.includedAssets = includedAssets; + } + + public boolean isIncludesLocales() { + return includesLocales; + } + + public void setIncludesLocales(boolean includesLocales) { + this.includesLocales = includesLocales; + } + + public List getIncludedLocales() { + return includedLocales; + } + + public void setIncludedLocales(List includedLocales) { + this.includedLocales = includedLocales; + } + @Deprecated public void changeElementId(String elementId) { this.elementId = elementId; diff --git a/streampipes-model/src/main/java/org/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/streampipes/model/connect/adapter/AdapterDescription.java index 9ba0ca761f..0e0bbeaceb 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/streampipes/model/connect/adapter/AdapterDescription.java @@ -31,6 +31,8 @@ import org.streampipes.model.grounding.SimpleTopicDefinition; import org.streampipes.model.grounding.TransportProtocol; import org.streampipes.model.staticproperty.StaticProperty; +import org.streampipes.model.util.Cloner; +import org.streampipes.vocabulary.StreamPipes; import java.util.ArrayList; import java.util.List; @@ -75,11 +77,17 @@ public abstract class AdapterDescription extends NamedStreamPipesEntity { @RdfProperty("sp:rules") private List rules; + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.ALL}) + @RdfProperty(StreamPipes.HAS_ADAPTER_TYPE) + private List category; + public AdapterDescription() { super(); this.rules = new ArrayList<>(); this.eventGrounding = new EventGrounding(); this.config = new ArrayList<>(); + this.category = new ArrayList<>(); // TODO move to another place TransportProtocol tp = new KafkaTransportProtocol(); @@ -93,6 +101,7 @@ public AdapterDescription() { public AdapterDescription(String uri, String name, String description) { super(uri, name, description); this.rules = new ArrayList<>(); + this.category = new ArrayList<>(); } @@ -103,6 +112,7 @@ public AdapterDescription(AdapterDescription other) { this.rules = other.getRules(); this.adapterType = other.getAdapterType(); this.icon = other.getIcon(); + this.category = new Cloner().epaTypes(other.getCategory()); if (other.getEventGrounding() != null) this.eventGrounding = new EventGrounding(other.getEventGrounding()); } @@ -211,6 +221,14 @@ public void setIcon(String icon) { this.icon = icon; } + public List getCategory() { + return category; + } + + public void setCategory(List category) { + this.category = category; + } + @Override public String toString() { return "AdapterDescription{" + diff --git a/streampipes-model/src/main/java/org/streampipes/model/connect/grounding/ProtocolDescription.java b/streampipes-model/src/main/java/org/streampipes/model/connect/grounding/ProtocolDescription.java index 8e4bd645a7..ca79997e4d 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/connect/grounding/ProtocolDescription.java +++ b/streampipes-model/src/main/java/org/streampipes/model/connect/grounding/ProtocolDescription.java @@ -23,13 +23,15 @@ import org.streampipes.model.base.NamedStreamPipesEntity; import org.streampipes.model.staticproperty.StaticProperty; import org.streampipes.model.util.Cloner; +import org.streampipes.vocabulary.StreamPipes; + +import java.util.ArrayList; +import java.util.List; import javax.persistence.CascadeType; import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.OneToMany; -import java.util.ArrayList; -import java.util.List; @Namespaces({"sp", "https://streampipes.org/vocabulary/v1/"}) @RdfsClass("sp:ProtocolDescription") @@ -47,25 +49,33 @@ public class ProtocolDescription extends NamedStreamPipesEntity { @RdfProperty("sp:config") List config; + @OneToMany(fetch = FetchType.EAGER, + cascade = {CascadeType.ALL}) + @RdfProperty(StreamPipes.HAS_ADAPTER_TYPE) + private List category; + public ProtocolDescription() { } public ProtocolDescription(String uri, String name, String description) { super(uri, name, description); this.config = new ArrayList<>(); + this.category = new ArrayList<>(); } public ProtocolDescription(String uri, String name, String description, List config) { super(uri, name, description); this.config = config; + this.category = new ArrayList<>(); } public ProtocolDescription(ProtocolDescription other) { super(other); this.config = new Cloner().staticProperties(other.getConfig()); - - + if (other.getCategory() != null) { + this.category = new Cloner().epaTypes(other.getCategory()); + } } public void addConfig(StaticProperty sp) { @@ -87,4 +97,12 @@ public String getSourceType() { public void setSourceType(String sourceType) { this.sourceType = sourceType; } + + public List getCategory() { + return category; + } + + public void setCategory(List category) { + this.category = category; + } } diff --git a/streampipes-model/src/main/java/org/streampipes/model/graph/DataProcessorInvocation.java b/streampipes-model/src/main/java/org/streampipes/model/graph/DataProcessorInvocation.java index fd396701b4..3a3954ab21 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/graph/DataProcessorInvocation.java +++ b/streampipes-model/src/main/java/org/streampipes/model/graph/DataProcessorInvocation.java @@ -47,7 +47,6 @@ public class DataProcessorInvocation extends InvocableStreamPipesEntity implemen @RdfProperty(StreamPipes.PRODUCES) private SpDataStream outputStream; - @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.ALL}) @RdfProperty(StreamPipes.HAS_OUTPUT_STRATEGY) diff --git a/streampipes-model/src/main/java/org/streampipes/model/schema/EventProperty.java b/streampipes-model/src/main/java/org/streampipes/model/schema/EventProperty.java index 9597edd387..929a3f3e41 100644 --- a/streampipes-model/src/main/java/org/streampipes/model/schema/EventProperty.java +++ b/streampipes-model/src/main/java/org/streampipes/model/schema/EventProperty.java @@ -79,6 +79,9 @@ public abstract class EventProperty extends UnnamedStreamPipesEntity { @RdfProperty(StreamPipes.HAS_PROPERTY_SCOPE) private String propertyScope; + @RdfProperty(StreamPipes.INDEX) + private int index = 0; + private String runtimeId; public EventProperty() { @@ -105,6 +108,7 @@ public EventProperty(EventProperty other) { this.domainProperties = other.getDomainProperties(); this.propertyScope = other.getPropertyScope(); this.runtimeId = other.getRuntimeId(); + this.index = other.getIndex(); } public EventProperty(List subClassOf) { @@ -208,4 +212,12 @@ public String getRuntimeId() { public void setRuntimeId(String runtimeId) { this.runtimeId = runtimeId; } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } } diff --git a/streampipes-performance-tests/pom.xml b/streampipes-performance-tests/pom.xml index ea009271f5..eb16b6edc6 100644 --- a/streampipes-performance-tests/pom.xml +++ b/streampipes-performance-tests/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-pipeline-management/pom.xml b/streampipes-pipeline-management/pom.xml index 440069bb8d..30bcc376e0 100644 --- a/streampipes-pipeline-management/pom.xml +++ b/streampipes-pipeline-management/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-pipeline-management diff --git a/streampipes-pipeline-management/src/main/java/org/streampipes/manager/endpoint/EndpointItemFetcher.java b/streampipes-pipeline-management/src/main/java/org/streampipes/manager/endpoint/EndpointItemFetcher.java index 9d098697cf..01a17f473f 100644 --- a/streampipes-pipeline-management/src/main/java/org/streampipes/manager/endpoint/EndpointItemFetcher.java +++ b/streampipes-pipeline-management/src/main/java/org/streampipes/manager/endpoint/EndpointItemFetcher.java @@ -57,7 +57,7 @@ private List getEndpointItems(RdfEndpoint e) { return new Gson().fromJson(result, new TypeToken>(){}.getType()); } catch (IOException e1) { - logger.warn("Processing Element Descriptions could not be fetched from RDF endpoint: " + e.getEndpointUrl(), e1); + logger.warn("Processing Element Descriptions could not be fetched from RDF endpoint: " + e.getEndpointUrl()); return new ArrayList<>(); } } diff --git a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/assets/TestImagePathReplacer.java b/streampipes-pipeline-management/src/test/java/org/streampipes/manager/assets/TestImagePathReplacer.java index f6ba9ae8c9..8318d3fa4c 100644 --- a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/assets/TestImagePathReplacer.java +++ b/streampipes-pipeline-management/src/test/java/org/streampipes/manager/assets/TestImagePathReplacer.java @@ -31,7 +31,7 @@ public class TestImagePathReplacer { "Lorem ipsu"; private String TEST_CONTENT_REPLACED = "## Numerical Filter\n" + "\n" + - "\n" + + "\n" + "\n" + "## Description\n" + "\n" + diff --git a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/TestPipelineValidationHandler.java b/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/TestPipelineValidationHandler.java index d370bcd280..868e08a203 100644 --- a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/TestPipelineValidationHandler.java +++ b/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/TestPipelineValidationHandler.java @@ -17,11 +17,9 @@ package org.streampipes.manager.matching; -import junit.framework.TestCase; - //import static org.assertj.core.api.Assertions.assertThat; -public class TestPipelineValidationHandler extends TestCase { +public class TestPipelineValidationHandler { // @Test // public void testPositivePipelineValidation() { diff --git a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/v2/TestStreamMatch.java b/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/v2/TestStreamMatch.java deleted file mode 100644 index 1ef7f97fec..0000000000 --- a/streampipes-pipeline-management/src/test/java/org/streampipes/manager/matching/v2/TestStreamMatch.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 FZI Forschungszentrum Informatik - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.streampipes.manager.matching.v2; - -import junit.framework.TestCase; - -public class TestStreamMatch extends TestCase { - -// @Test -// public void testPositiveStreamMatchWithIgnoredGrounding() { -// -// DataProcessorDescription requiredSepa = new AggregationController().declareModel(); -// SpDataStream offeredStream = new RandomNumberStreamJson().declareModel(new RandomDataProducer().declareModel()); -// -// SpDataStream requiredStream = requiredSepa.getSpDataStreams().get(0); -// -// List errorLog = new ArrayList<>(); -// -// boolean matches = new StreamMatch().matchIgnoreGrounding(offeredStream, requiredStream, errorLog); -// assertTrue(matches); -// } - -// @Test -// public void testPositiveStreamMatchWithoutRequirementsIgnoredGrounding() { -// -// DataProcessorDescription requiredSepa = new ProjectController().declareModel(); -// SpDataStream offeredStream = new RandomNumberStreamJson().declareModel(new RandomDataProducer().declareModel()); -// -// SpDataStream requiredStream = requiredSepa.getSpDataStreams().get(0); -// -// List errorLog = new ArrayList<>(); -// -// boolean matches = new StreamMatch().matchIgnoreGrounding(offeredStream, requiredStream, errorLog); -// assertTrue(matches); -// } -} diff --git a/streampipes-rest-shared/pom.xml b/streampipes-rest-shared/pom.xml index 960e364409..8cc816da5b 100644 --- a/streampipes-rest-shared/pom.xml +++ b/streampipes-rest-shared/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-rest-shared/src/test/java/org/streampipes/rest/shared/util/JsonLdUtilsTest.java b/streampipes-rest-shared/src/test/java/org/streampipes/rest/shared/util/JsonLdUtilsTest.java index 22db3a6806..96a0a07025 100644 --- a/streampipes-rest-shared/src/test/java/org/streampipes/rest/shared/util/JsonLdUtilsTest.java +++ b/streampipes-rest-shared/src/test/java/org/streampipes/rest/shared/util/JsonLdUtilsTest.java @@ -17,17 +17,16 @@ package org.streampipes.rest.shared.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import org.junit.Test; -import org.streampipes.model.connect.adapter.*; import org.streampipes.model.connect.grounding.FormatDescription; import org.streampipes.model.connect.grounding.FormatDescriptionList; -import org.streampipes.vocabulary.StreamPipes; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; - public class JsonLdUtilsTest { @Test @@ -42,25 +41,25 @@ public void fromToJsonLdSimple() { assertEquals(result.getList().size(), 0); } - @Test - public void fromToJsonLdSimpleComplexTopElement() { - List list = Arrays.asList( - new GenericAdapterStreamDescription(), - new GenericAdapterSetDescription()); - - AdapterDescriptionList object = new AdapterDescriptionList(list); - String jsonLD = JsonLdUtils.toJsonLD(object); - - AdapterDescriptionList result = JsonLdUtils.fromJsonLd(jsonLD, AdapterDescriptionList.class, StreamPipes.ADAPTER_DESCRIPTION_LIST); - - assertEquals(result.getUri(), "http://streampipes.org/adapterlist"); - assertNotNull(result.getList()); - assertEquals(2, result.getList().size()); - assertEquals("http://id.de#3", result.getList().get(0).getUri()); - assertEquals("name1", result.getList().get(0).getName()); - assertEquals("http://id.de#4", result.getList().get(1).getUri()); - assertEquals("name2", result.getList().get(1).getName()); - } +// @Test +// public void fromToJsonLdSimpleComplexTopElement() { +// List list = Arrays.asList( +// new GenericAdapterStreamDescription(), +// new GenericAdapterSetDescription()); +// +// AdapterDescriptionList object = new AdapterDescriptionList(list); +// String jsonLD = JsonLdUtils.toJsonLD(object); +// +// AdapterDescriptionList result = JsonLdUtils.fromJsonLd(jsonLD, AdapterDescriptionList.class, StreamPipes.ADAPTER_DESCRIPTION_LIST); +// +// assertEquals(result.getUri(), "http://streampipes.org/adapterlist"); +// assertNotNull(result.getList()); +// assertEquals(2, result.getList().size()); +// assertEquals("http://id.de#3", result.getList().get(0).getUri()); +// assertEquals("name1", result.getList().get(0).getName()); +// assertEquals("http://id.de#4", result.getList().get(1).getUri()); +// assertEquals("name2", result.getList().get(1).getName()); +// } @Test public void fromToJsonLdComplex() { diff --git a/streampipes-rest/pom.xml b/streampipes-rest/pom.xml index 8e5aa81c4c..d300388bc6 100644 --- a/streampipes-rest/pom.xml +++ b/streampipes-rest/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 StreamPipes REST API streampipes-rest @@ -79,7 +79,12 @@ org.elasticsearch.client elasticsearch-rest-high-level-client - 6.2.1 + 6.6.2 + + + org.influxdb + influxdb-java + 2.14 \ No newline at end of file diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/api/IAssetDashboard.java b/streampipes-rest/src/main/java/org/streampipes/rest/api/IAssetDashboard.java new file mode 100644 index 0000000000..2d2bf030d4 --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/api/IAssetDashboard.java @@ -0,0 +1,39 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.rest.api; + +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.streampipes.model.client.assetdashboard.AssetDashboardConfig; + +import java.io.InputStream; + +import javax.ws.rs.core.Response; + +public interface IAssetDashboard { + + Response getAssetDashboard(String dashboardId); + + Response getAllDashboards(); + + Response storeAssetDashboard(AssetDashboardConfig dashboardConfig); + + Response deleteAssetDashboard(String dashboardId); + + Response storeDashboardImage(InputStream uploadedInputStream, + FormDataContentDisposition fileDetail); + + Response getDashboardImage(String imageName); +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/api/IPipelineElementCategory.java b/streampipes-rest/src/main/java/org/streampipes/rest/api/IPipelineElementCategory.java index 1f1e8a93ba..64408e7e15 100644 --- a/streampipes-rest/src/main/java/org/streampipes/rest/api/IPipelineElementCategory.java +++ b/streampipes-rest/src/main/java/org/streampipes/rest/api/IPipelineElementCategory.java @@ -21,10 +21,12 @@ public interface IPipelineElementCategory { - Response getEps(); + Response getEps(); - Response getEpaCategories(); + Response getEpaCategories(); + + Response getEcCategories(); + + Response getAdapterCategories(); - Response getEcCategories(); - } diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/application/StreamPipesApplication.java b/streampipes-rest/src/main/java/org/streampipes/rest/application/StreamPipesApplication.java index 7c0347a0da..81cd8f7b71 100644 --- a/streampipes-rest/src/main/java/org/streampipes/rest/application/StreamPipesApplication.java +++ b/streampipes-rest/src/main/java/org/streampipes/rest/application/StreamPipesApplication.java @@ -17,37 +17,9 @@ package org.streampipes.rest.application; -import org.streampipes.rest.impl.ApplicationLink; -import org.streampipes.rest.impl.Authentication; -import org.streampipes.rest.impl.AutoComplete; -import org.streampipes.rest.impl.ConsulConfig; -import org.streampipes.rest.impl.ContainerProvidedOptions; -import org.streampipes.rest.impl.Couchdb; -import org.streampipes.rest.impl.DataStream; -import org.streampipes.rest.impl.Deployment; -import org.streampipes.rest.impl.InternalPipelineTemplates; -import org.streampipes.rest.impl.Notification; -import org.streampipes.rest.impl.OntologyContext; -import org.streampipes.rest.impl.OntologyKnowledge; -import org.streampipes.rest.impl.OntologyMeasurementUnit; -import org.streampipes.rest.impl.OntologyPipelineElement; -import org.streampipes.rest.impl.PipelineCategory; -import org.streampipes.rest.impl.PipelineElementAsset; -import org.streampipes.rest.impl.PipelineElementCategory; -import org.streampipes.rest.impl.PipelineElementImport; -import org.streampipes.rest.impl.PipelineElementRuntimeInfo; -import org.streampipes.rest.impl.PipelineTemplate; -import org.streampipes.rest.impl.PipelineWithUserResource; -import org.streampipes.rest.impl.RdfEndpoint; -import org.streampipes.rest.impl.SemanticEventConsumer; -import org.streampipes.rest.impl.SemanticEventProcessingAgent; -import org.streampipes.rest.impl.SemanticEventProducer; -import org.streampipes.rest.impl.Setup; -import org.streampipes.rest.impl.StreamPipesLogs; -import org.streampipes.rest.impl.User; -import org.streampipes.rest.impl.Version; -import org.streampipes.rest.impl.VirtualSensor; -import org.streampipes.rest.impl.Visualization; +import org.streampipes.rest.impl.*; +import org.streampipes.rest.impl.datalake.DataLakeResource; +import org.streampipes.rest.impl.datalake.DataLakeResourceV3; import org.streampipes.rest.impl.nouser.PipelineElementImportNoUser; import org.streampipes.rest.impl.nouser.PipelineNoUserResource; import org.streampipes.rest.shared.serializer.GsonClientModelProvider; @@ -55,11 +27,10 @@ import org.streampipes.rest.shared.serializer.GsonWithoutIdProvider; import org.streampipes.rest.shared.serializer.JsonLdProvider; +import javax.ws.rs.core.Application; import java.util.HashSet; import java.util.Set; -import javax.ws.rs.core.Application; - public class StreamPipesApplication extends Application { @Override @@ -68,6 +39,7 @@ public Set> getClasses() { // APIs apiClasses.add(Authentication.class); + apiClasses.add(AssetDashboard.class); apiClasses.add(AutoComplete.class); apiClasses.add(PipelineElementCategory.class); apiClasses.add(Deployment.class); @@ -80,6 +52,7 @@ public Set> getClasses() { apiClasses.add(PipelineNoUserResource.class); apiClasses.add(PipelineElementImportNoUser.class); apiClasses.add(PipelineCategory.class); + apiClasses.add(DataLakeResource.class); apiClasses.add(PipelineElementImport.class); apiClasses.add(SemanticEventConsumer.class); apiClasses.add(SemanticEventProcessingAgent.class); @@ -100,6 +73,7 @@ public Set> getClasses() { apiClasses.add(PipelineElementRuntimeInfo.class); apiClasses.add(Version.class); apiClasses.add(PipelineElementAsset.class); + apiClasses.add(DataLakeResourceV3.class); // Serializers diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/AbstractRestInterface.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/AbstractRestInterface.java index 8aec1b4d2a..b41f04a7f0 100644 --- a/streampipes-rest/src/main/java/org/streampipes/rest/impl/AbstractRestInterface.java +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/AbstractRestInterface.java @@ -170,6 +170,10 @@ protected Response ok(T entity) { .build(); } + protected Response ok() { + return Response.ok().build(); + } + protected Response fail() { return Response.serverError().build(); } diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/AssetDashboard.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/AssetDashboard.java new file mode 100644 index 0000000000..58e54b06cd --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/AssetDashboard.java @@ -0,0 +1,125 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.rest.impl; + +import org.apache.commons.io.FileUtils; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.streampipes.model.client.assetdashboard.AssetDashboardConfig; +import org.streampipes.rest.api.IAssetDashboard; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.FileNameMap; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Paths; + +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/v2/users/{username}/asset-dashboards") +public class AssetDashboard extends AbstractRestInterface implements IAssetDashboard { + + private static final String APP_ID = "org.streampipes.apps.assetdashboard"; + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Path("/{dashboardId}") + @Override + public Response getAssetDashboard(@PathParam("dashboardId") String dashboardId) { + return ok(getNoSqlStorage().getAssetDashboardStorage().getAssetDashboard(dashboardId)); + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Override + public Response getAllDashboards() { + return ok(getNoSqlStorage().getAssetDashboardStorage().getAllAssetDashboards()); + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Override + public Response storeAssetDashboard(AssetDashboardConfig dashboardConfig) { + getNoSqlStorage().getAssetDashboardStorage().storeAssetDashboard(dashboardConfig); + return ok(); + } + + @DELETE + @Produces(MediaType.APPLICATION_JSON) + @Path("/{dashboardId}") + @Override + public Response deleteAssetDashboard(@PathParam("dashboardId") String dashboardId) { + getNoSqlStorage().getAssetDashboardStorage().deleteAssetDashboard(dashboardId); + return ok(); + } + + @GET + @Path("/images/{imageName}") + @Override + public Response getDashboardImage(@PathParam("imageName") String imageName) { + try { + java.nio.file.Path path = Paths.get(getTargetFile(imageName)); + File file = new File(path.toString()); + FileNameMap fileNameMap = URLConnection.getFileNameMap(); + String mimeType = fileNameMap.getContentTypeFor(file.getName()); + return Response.ok(Files.readAllBytes(path)).type(mimeType).build(); + } catch (IOException e) { + e.printStackTrace(); + return fail(); + } + } + + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Path("/images") + @Override + public Response storeDashboardImage(@FormDataParam("file_upload") InputStream uploadedInputStream, + @FormDataParam("file_upload") FormDataContentDisposition fileDetail) { + File targetDirectory = new File(getTargetDirectory()); + if (!targetDirectory.exists()) { + targetDirectory.mkdirs(); + } + + File targetFile = new File(getTargetFile(fileDetail.getFileName())); + + try { + FileUtils.copyInputStreamToFile(uploadedInputStream, targetFile); + return ok(); + } catch (IOException e) { + e.printStackTrace(); + return fail(); + } + } + + private String getTargetDirectory() { + return System.getProperty("user.home") + File.separator + ".streampipes" + + File.separator + "assets" + File.separator + APP_ID; + } + + private String getTargetFile(String filename) { + return getTargetDirectory() + File.separator + filename; + } +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementAsset.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementAsset.java index 48e085e2cb..725d2e0db5 100644 --- a/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementAsset.java +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementAsset.java @@ -37,7 +37,6 @@ public Response getIconAsset(@PathParam("appId") String appId) { try { return ok(AssetManager.getAssetIcon(appId)); } catch (IOException e) { - e.printStackTrace(); return fail(); } } @@ -49,7 +48,6 @@ public Response getDocumentationAsset(@PathParam("appId") String appId) { try { return ok(AssetManager.getAssetDocumentation(appId)); } catch (IOException e) { - e.printStackTrace(); return fail(); } } diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementCategory.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementCategory.java index b6d8136bca..f7bd7dc9e4 100644 --- a/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementCategory.java +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/PipelineElementCategory.java @@ -17,6 +17,7 @@ package org.streampipes.rest.impl; +import org.streampipes.model.AdapterType; import org.streampipes.model.DataProcessorType; import org.streampipes.model.DataSinkType; import org.streampipes.model.client.Category; @@ -51,6 +52,14 @@ public Response getEpaCategories() { return ok(DataProcessorType.values()); } + @GET + @Path("/adapter") + @Produces("application/json") + @Override + public Response getAdapterCategories() { + return ok(AdapterType.values()); + } + @GET @Path("/ec") @Produces("application/json") diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagement.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagement.java new file mode 100644 index 0000000000..4c846e4393 --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagement.java @@ -0,0 +1,147 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake; + +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.search.*; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.streampipes.config.backend.BackendConfig; +import org.streampipes.rest.impl.datalake.model.DataResult; +import org.streampipes.rest.impl.datalake.model.InfoResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class DataLakeManagement { + + public DataResult getEvents(String index) throws IOException { + List> events = getDataLakeData(index); + DataResult dataResult = new DataResult(events.size(), events); + return dataResult; + } + + public DataResult getEvents(String index, String timestampRuntimeName, Long from, Long to) throws IOException { + List> events = getDataLakeData(index, timestampRuntimeName, from, to); + DataResult dataResult = new DataResult(events.size(), events); + return dataResult; + } + + public InfoResult getInfo(String index) { + return null; + } + + public List getAllInfos() { + return null; + } + + + public List> getDataLakeData(String index) throws IOException { + return getDataLakeData(index, null, -1, -1); + } + + public List> getDataLakeData(String index, String timestampRuntimeName, long from, long to) throws IOException { + List> result = new ArrayList<>(); + + RestHighLevelClient client = getRestHighLevelClient(); + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + if (timestampRuntimeName != null) { + searchSourceBuilder.query(QueryBuilders.rangeQuery(timestampRuntimeName).from(from).to(to)); + } + + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest); + String scrollId = searchResponse.getScrollId(); + + + SearchHit[] searchHits = searchResponse.getHits().getHits(); + + for (SearchHit hit : searchHits) { + result.add(hit.getSourceAsMap()); + } + + while (searchHits != null && searchHits.length > 0) { + + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + for (SearchHit hit : searchHits) { + result.add(hit.getSourceAsMap()); + } + } + + + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + + return result; + } + + private RestHighLevelClient getRestHighLevelClient() { + String host = BackendConfig.INSTANCE.getDatalakeHost(); + int port = BackendConfig.INSTANCE.getDatalakePort(); + + return new RestHighLevelClient( + RestClient.builder( + new HttpHost(host, port, "http"))); + } + + public String deleteIndex(String index) throws IOException { + RestHighLevelClient client = getRestHighLevelClient(); + + DeleteIndexRequest request = new DeleteIndexRequest(index); + + try { + AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); + if (deleteIndexResponse.isAcknowledged()) { + return ""; + } else { + return "Index: " + index + " did not exist!"; + } + + } catch (ElasticsearchException exception) { + if (exception.status() == RestStatus.NOT_FOUND) { + return "Index: " + index + " did not exist!"; + } + } + + return "Index: " + index + " did not exist!"; + } + +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagementV3.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagementV3.java new file mode 100644 index 0000000000..2c63a2c171 --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeManagementV3.java @@ -0,0 +1,193 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake; + +import com.google.gson.Gson; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.streampipes.config.backend.BackendConfig; +import org.streampipes.rest.impl.datalake.model.InfoResult; +import org.streampipes.rest.impl.datalake.model.PageResult; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.StreamingOutput; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DataLakeManagementV3 { + + public List getInfos() { + List indices = new ArrayList<>(); + InfluxDB influxDB = getInfluxDBClient(); + + Query query = new Query("SHOW MEASUREMENTS ON " + BackendConfig.INSTANCE.getInfluxDatabaseName(), + BackendConfig.INSTANCE.getInfluxDatabaseName()); + QueryResult result = influxDB.query(query); + + result.getResults().get(0).getSeries().get(0).getValues().forEach(value -> + indices.add(new InfoResult((String) value.get(0), null)) + ); + + influxDB.close(); + return indices; + } + + public PageResult getEvents(String index, int itemsPerPage) throws IOException { + int page = getMaxPage(index, itemsPerPage); + return getEvents(index, itemsPerPage, page); + } + + public PageResult getEvents(String index, int itemsPerPage, int page) throws IOException { + InfluxDB influxDB = getInfluxDBClient(); + Query query = new Query("SELECT * FROM " + index + " ORDER BY time LIMIT " + itemsPerPage + " OFFSET " + page * itemsPerPage, + BackendConfig.INSTANCE.getInfluxDatabaseName()); + QueryResult result = influxDB.query(query); + + List> events = new ArrayList<>(); + if(result.getResults().get(0).getSeries() != null) { + events = convertResult(result.getResults().get(0).getSeries().get(0)); + } + influxDB.close(); + + int pageSum = getMaxPage(index, itemsPerPage); + + return new PageResult(events.size(), events, page, pageSum); + } + + public StreamingOutput getAllEvents(String index, String outputFormat) { + return new StreamingOutput() { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException { + InfluxDB influxDB = getInfluxDBClient(); + int itemsPerRequest = 200; + + //JSON + if (outputFormat.equals("json")) { + int i = 0; + boolean isFirstElement = true; + List> convertResult = new ArrayList<>(); + Gson gson = new Gson(); + + do { + outputStream.write(toBytes("[")); + + Query query = new Query("SELECT * FROM " + index + " ORDER BY time LIMIT " + itemsPerRequest + " OFFSET " + i * itemsPerRequest, + BackendConfig.INSTANCE.getInfluxDatabaseName()); + QueryResult result = influxDB.query(query); + if((result.getResults().get(0).getSeries() != null)) + convertResult = convertResult(result.getResults().get(0).getSeries().get(0)); + + for (Map event : convertResult) { + if (!isFirstElement) + outputStream.write(toBytes(",")); + isFirstElement = false; + outputStream.write(toBytes(gson.toJson(event))); + } + convertResult = new ArrayList<>(); + i++; + } while (convertResult.size() > 0); + outputStream.write(toBytes("]")); + + //CSV + } else if (outputFormat.equals("csv")) { + int i = 0; + + Query query = new Query("SELECT * FROM " + index + " ORDER BY time LIMIT " + itemsPerRequest + " OFFSET " + i * itemsPerRequest, + BackendConfig.INSTANCE.getInfluxDatabaseName()); + QueryResult result = influxDB.query(query); + if((result.getResults().get(0).getSeries() != null)) { + //HEADER + QueryResult.Series serie = result.getResults().get(0).getSeries().get(0); + for (int i2 = 0; i2 < serie.getColumns().size(); i2++) { + outputStream.write(toBytes(serie.getColumns().get(i2))); + if(i2 < serie.getColumns().size() -1) + outputStream.write(toBytes(";")); + } + outputStream.write(toBytes("\n")); + } + + boolean newResults; + do { + newResults = false; + query = new Query("SELECT * FROM " + index + " ORDER BY time LIMIT " + itemsPerRequest + " OFFSET " + i * itemsPerRequest, + BackendConfig.INSTANCE.getInfluxDatabaseName()); + result = influxDB.query(query); + if((result.getResults().get(0).getSeries() != null)) { + newResults = true; + QueryResult.Series serie = result.getResults().get(0).getSeries().get(0); + for (int i2 = 0; i2 < serie.getValues().size() - 1; i2++) { + for (int i3 = 0; i3 < serie.getValues().get(i2).size(); i3++) { + outputStream.write(toBytes(serie.getValues().get(i2).get(i3).toString())); + if(i3 < serie.getValues().get(i2).size() - 1) + outputStream.write(toBytes(";")); + } + outputStream.write(toBytes("\n")); + } + } + i++; + } while (newResults); + + } + } + }; + } + + private byte[] toBytes(String value) { + return value.getBytes(); + } + + private int getMaxPage(String index, int itemsPerPage) { + InfluxDB influxDB = getInfluxDBClient(); + Query query = new Query("SELECT count(*) FROM " + index, BackendConfig.INSTANCE.getInfluxDatabaseName()); + QueryResult result = influxDB.query(query); + + + int size = ((Double) result.getResults().get(0).getSeries().get(0).getValues().get(0).get(1)).intValue() ; + int page = size / itemsPerPage; + + influxDB.close(); + return page; + } + + private List> convertResult(QueryResult.Series serie) { + List> events = new ArrayList<>(); + List columns = serie.getColumns(); + + for (List value : serie.getValues()) { + Map event = new HashMap<>(); + for (int i = 0; i < value.size(); i++) { + event.put(columns.get(i), value.get(i)); + } + events.add(event); + } + return events; + + } + + + private InfluxDB getInfluxDBClient() { + return InfluxDBFactory.connect(BackendConfig.INSTANCE.getInfluxUrl()); + } + +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResource.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResource.java new file mode 100644 index 0000000000..e88ef8b079 --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResource.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake; + + +import org.streampipes.rest.impl.AbstractRestInterface; +import org.streampipes.rest.impl.datalake.model.DataResult; +import org.streampipes.rest.impl.datalake.model.InfoResult; +import org.streampipes.rest.shared.annotation.GsonWithIds; + +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.io.IOException; +import java.util.List; + + +@Path("/v2/users/{username}/datalake") +public class DataLakeResource extends AbstractRestInterface { + private DataLakeManagement dataLakeManagement; + + public DataLakeResource() { + this.dataLakeManagement = new DataLakeManagement(); + } + + public DataLakeResource(DataLakeManagement dataLakeManagement) { + this.dataLakeManagement = dataLakeManagement; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/data/{index}") + public Response getAllData(@Context UriInfo info, @PathParam("index") String index) { + + DataResult result; + String from = info.getQueryParameters().getFirst("from"); + String to = info.getQueryParameters().getFirst("to"); + String timestamp = info.getQueryParameters().getFirst("timestamp"); + + try { + if (from != null && to != null && timestamp != null) { + result = this.dataLakeManagement.getEvents(index, timestamp, Long.parseLong(from), Long.parseLong(to)); + return Response.ok(result).build(); + + } else { + result = this.dataLakeManagement.getEvents(index); + return Response.ok(result).build(); + } + } catch (IOException e) { + e.printStackTrace(); + + return Response.serverError().build(); + } + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/delete/{index}") + public Response delteIndex(@PathParam("index") String index) { + + try { + String result = this.dataLakeManagement.deleteIndex(index); + if (result.equals("")) { + return Response.ok(result).build(); + } else { + + return Response.serverError().build(); + } + + } catch(IOException e){ + e.printStackTrace(); + + return Response.serverError().build(); + } + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/info/{index}") + public Response getInfo(@PathParam("index") String index) { + InfoResult result = this.dataLakeManagement.getInfo(index); + + return Response.ok(result).build(); + + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/info") + public Response getAllInfos() { + List result = this.dataLakeManagement.getAllInfos(); + + return Response.ok(result).build(); + } + +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResourceV3.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResourceV3.java new file mode 100644 index 0000000000..1f27c9e55f --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/DataLakeResourceV3.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake; + + +import org.streampipes.rest.impl.AbstractRestInterface; +import org.streampipes.rest.impl.datalake.model.InfoResult; +import org.streampipes.rest.impl.datalake.model.PageResult; +import org.streampipes.rest.shared.annotation.GsonWithIds; + +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import java.io.IOException; +import java.util.List; + + +@Path("/v3/users/{username}/datalake") +public class DataLakeResourceV3 extends AbstractRestInterface { + private DataLakeManagementV3 dataLakeManagement; + + public DataLakeResourceV3() { + this.dataLakeManagement = new DataLakeManagementV3(); + } + + public DataLakeResourceV3(DataLakeManagementV3 dataLakeManagement) { + this.dataLakeManagement = dataLakeManagement; + } + + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/data/{index}/paging") + public Response getPage(@PathParam("index") String index, + @Context UriInfo info, + @QueryParam("itemsPerPage") int itemsPerPage) { + + PageResult result; + String page = info.getQueryParameters().getFirst("page"); + + try { + + if(page != null) { + result = this.dataLakeManagement.getEvents(index, itemsPerPage, Integer.parseInt(page)); + } else { + result = this.dataLakeManagement.getEvents(index, itemsPerPage); + } + return Response.ok(result).build(); + } catch (IOException e) { + e.printStackTrace(); + + return Response.serverError().build(); + } + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/info") + public Response getAllInfos() { + List result = this.dataLakeManagement.getInfos(); + + return Response.ok(result).build(); + } + + @GET + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @Path("/data/{index}") + public Response getAllData(@PathParam("index") String index, @QueryParam("format") String format) { + StreamingOutput streamingOutput = dataLakeManagement.getAllEvents(index, format); + + return Response.ok(streamingOutput, MediaType.APPLICATION_OCTET_STREAM). + header("Content-Disposition", "attachment; filename=\"datalake" + format + "\"") + .build(); + } + +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/DataResult.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/DataResult.java new file mode 100644 index 0000000000..d2ae6218cd --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/DataResult.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake.model; + +import java.util.List; +import java.util.Map; + +public class DataResult { + + private int total; + private List> events; + + public DataResult() { + } + + public DataResult(int total, List> events) { + this.total = total; + this.events = events; + } + + public int getTotal() { + return total; + } + + public void setTotal(int total) { + this.total = total; + } + + public List> getEvents() { + return events; + } + + public void setEvents(List> events) { + this.events = events; + } +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/InfoResult.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/InfoResult.java new file mode 100644 index 0000000000..4807d3adfb --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/InfoResult.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.datalake.model; + +import org.streampipes.model.schema.EventSchema; + +public class InfoResult { + + private String index; + private EventSchema eventSchema; + + public InfoResult() { + } + + public InfoResult(String index, EventSchema eventSchema) { + this.index = index; + this.eventSchema = eventSchema; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public EventSchema getEventSchema() { + return eventSchema; + } + + public void setEventSchema(EventSchema eventSchema) { + this.eventSchema = eventSchema; + } +} diff --git a/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/PageResult.java b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/PageResult.java new file mode 100644 index 0000000000..5b2eda1eb6 --- /dev/null +++ b/streampipes-rest/src/main/java/org/streampipes/rest/impl/datalake/model/PageResult.java @@ -0,0 +1,49 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.rest.impl.datalake.model; + +import java.util.List; +import java.util.Map; + +public class PageResult extends DataResult { + + private int page; + + private int pageSum; + + public PageResult(int total, List> events, int page, int pageSum) { + super(total, events); + this.page = page; + this.pageSum = pageSum; + } + + public int getPage() { + return page; + } + + public void setPage(int page) { + this.page = page; + } + + public int getPageSum() { + return pageSum; + } + + public void setPageSum(int pageSum) { + this.pageSum = pageSum; + } +} diff --git a/streampipes-sdk/pom.xml b/streampipes-sdk/pom.xml index 2b8682e2ac..73bfc1a594 100644 --- a/streampipes-sdk/pom.xml +++ b/streampipes-sdk/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractConfigurablePipelineElementBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractConfigurablePipelineElementBuilder.java index f47c011a74..ecb74feaf5 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractConfigurablePipelineElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractConfigurablePipelineElementBuilder.java @@ -46,6 +46,11 @@ protected AbstractConfigurablePipelineElementBuilder(String appId, String label, this.staticProperties = new ArrayList<>(); } + protected AbstractConfigurablePipelineElementBuilder(String appId, T element) { + super(appId, element); + this.staticProperties = new ArrayList<>(); + } + /** * * @param staticProperty: The required static property (e.g., user input as shown in the StreamPipes UI diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractPipelineElementBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractPipelineElementBuilder.java index ffd094143d..6612fbb5d9 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractPipelineElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractPipelineElementBuilder.java @@ -20,43 +20,71 @@ import org.streampipes.model.base.NamedStreamPipesEntity; import org.streampipes.model.staticproperty.StaticProperty; import org.streampipes.sdk.helpers.Label; +import org.streampipes.sdk.helpers.Locales; + +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.Stream; public abstract class AbstractPipelineElementBuilder, T extends NamedStreamPipesEntity> { - protected T elementDescription; + protected T elementDescription; + + protected AbstractPipelineElementBuilder(String appId, String label, String description, T element) { + this(appId, element); + this.elementDescription.setName(label); + this.elementDescription.setDescription(description); + } + + protected AbstractPipelineElementBuilder(String appId, T element) { + this.elementDescription = element; + this.elementDescription.setElementId(appId); + this.elementDescription.setAppId(appId); + } + + public BU iconUrl(String iconUrl) { + elementDescription.setIconUrl(iconUrl); + return me(); + } + + @Deprecated + /** + * @deprecated: Use {@link #withAssets(String...)} instead + */ + public BU providesAssets(String... assets) { + return withAssets(assets); + } - protected AbstractPipelineElementBuilder(String appId, String label, String description, T element) { - this.elementDescription = element; - this.elementDescription.setElementId(appId); - this.elementDescription.setAppId(appId); - this.elementDescription.setName(label); - this.elementDescription.setDescription(description); - } + public BU withAssets(String... assets) { + this.elementDescription.setIncludesAssets(true); + this.elementDescription.setIncludedAssets(Arrays.asList(assets)); + return me(); + } - public BU iconUrl(String iconUrl) { - elementDescription.setIconUrl(iconUrl); - return me(); - } + public BU withLocales(Locales... locales) { + this.elementDescription.setIncludesLocales(true); + this.elementDescription.setIncludedLocales(Stream + .of(locales) + .map(Locales::toFilename) + .collect(Collectors.toList())); - public BU providesAssets() { - this.elementDescription.setIncludesAssets(true); - return me(); - } + return me(); + } - protected SP prepareStaticProperty(Label label, SP element) { - element.setInternalName(label.getInternalId()); - element.setDescription(label.getDescription()); - element.setLabel(label.getLabel()); + protected SP prepareStaticProperty(Label label, SP element) { + element.setInternalName(label.getInternalId()); + element.setDescription(label.getDescription()); + element.setLabel(label.getLabel()); - return element; - } + return element; + } - protected abstract BU me(); + protected abstract BU me(); - protected abstract void prepareBuild(); + protected abstract void prepareBuild(); - public T build() { - prepareBuild(); - return elementDescription; - } + public T build() { + prepareBuild(); + return elementDescription; + } } diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractProcessingElementBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractProcessingElementBuilder.java index d916e42ea4..2924c42cc8 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractProcessingElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/AbstractProcessingElementBuilder.java @@ -60,6 +60,14 @@ protected AbstractProcessingElementBuilder(String id, String label, String descr this.supportedGrounding = new EventGrounding(); } + protected AbstractProcessingElementBuilder(String id, T element) { + super(id, element); + this.streamRequirements = new ArrayList<>(); + this.stream1Properties = new ArrayList<>(); + this.stream2Properties = new ArrayList<>(); + this.supportedGrounding = new EventGrounding(); + } + /** * @deprecated Use {@link #requiredStream(CollectedStreamRequirements)} instead */ @@ -278,6 +286,11 @@ public void prepareBuild() { } this.elementDescription.setSupportedGrounding(supportedGrounding); + + for(int i = 0; i < streamRequirements.size(); i++) { + streamRequirements.get(i).setIndex(i); + } + this.elementDescription.setSpDataStreams(streamRequirements); } diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataSinkBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataSinkBuilder.java index ef5fa0e94b..40f343125e 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataSinkBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataSinkBuilder.java @@ -19,38 +19,55 @@ import org.streampipes.model.DataSinkType; import org.streampipes.model.graph.DataSinkDescription; +import org.streampipes.sdk.helpers.Locales; import java.util.Arrays; import java.util.stream.Collectors; public class DataSinkBuilder extends AbstractProcessingElementBuilder { - protected DataSinkBuilder(String id, String label, String description) { - super(id, label, description, new DataSinkDescription()); - } - - /** - * Creates a new data sink using the builder pattern. - * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink - * @param label A human-readable name of the element. Will later be shown as the element name in the StreamPipes UI. - * @param description A human-readable description of the element. - */ - public static DataSinkBuilder create(String id, String label, String description) - { - return new DataSinkBuilder(id, label, description); - } - - public DataSinkBuilder category(DataSinkType... categories) { - this.elementDescription - .setCategory(Arrays - .stream(categories) - .map(Enum::name) - .collect(Collectors.toList())); - return me(); - } - - @Override - protected DataSinkBuilder me() { - return this; - } + protected DataSinkBuilder(String id, String label, String description) { + super(id, label, description, new DataSinkDescription()); + } + + protected DataSinkBuilder(String id) { + super(id, new DataSinkDescription()); + } + + /** + * Creates a new data sink using the builder pattern. + * + * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink + * @param label A human-readable name of the element. Will later be shown as the element name in the StreamPipes UI. + * @param description A human-readable description of the element. + */ + public static DataSinkBuilder create(String id, String label, String description) { + return new DataSinkBuilder(id, label, description); + } + + /** + * Creates a new data sink using the builder pattern. If no label and description is given + * for an element, + * {@link org.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * must be called. + * + * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink + */ + public static DataSinkBuilder create(String id) { + return new DataSinkBuilder(id); + } + + public DataSinkBuilder category(DataSinkType... categories) { + this.elementDescription + .setCategory(Arrays + .stream(categories) + .map(Enum::name) + .collect(Collectors.toList())); + return me(); + } + + @Override + protected DataSinkBuilder me() { + return this; + } } diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataStreamBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataStreamBuilder.java index 9f23dba32a..2771fa6e54 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataStreamBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/DataStreamBuilder.java @@ -105,6 +105,10 @@ protected DataStreamBuilder me() { @Override protected void prepareBuild() { this.elementDescription.setEventGrounding(eventGrounding); + + for (int i = 0; i < eventProperties.size(); i++) { + eventProperties.get(i).setIndex(i); + } this.elementDescription.setEventSchema(new EventSchema(eventProperties)); } } diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/ProcessingElementBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/ProcessingElementBuilder.java index 96bb249b5f..33ebfcb42b 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/ProcessingElementBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/ProcessingElementBuilder.java @@ -21,6 +21,7 @@ import org.streampipes.model.graph.DataProcessorDescription; import org.streampipes.model.output.OutputStrategy; import org.streampipes.sdk.helpers.Label; +import org.streampipes.sdk.helpers.Locales; import java.util.ArrayList; import java.util.Arrays; @@ -37,6 +38,11 @@ private ProcessingElementBuilder(String id, String name, String description) { this.outputStrategies = new ArrayList<>(); } + private ProcessingElementBuilder(String id) { + super(id, new DataProcessorDescription()); + this.outputStrategies = new ArrayList<>(); + } + /** * Creates a new processing element using the builder pattern. * @param id A unique identifier of the new element, e.g., com.mycompany.processor.mynewdataprocessor @@ -51,6 +57,19 @@ public static ProcessingElementBuilder create(Label label) { return new ProcessingElementBuilder(label.getInternalId(), label.getLabel(), label.getDescription()); } + /** + * Creates a new processing element using the builder pattern. If no label and description is + * given + * for an element, + * {@link org.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * must be called. + * + * @param id A unique identifier of the new element, e.g., com.mycompany.sink.mynewdatasink + */ + public static ProcessingElementBuilder create(String id) { + return new ProcessingElementBuilder(id); + } + /** * Assigns an output strategy to the element which defines the output the data processor produces. * @param outputStrategy An {@link org.streampipes.model.output.OutputStrategy}. Use diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java index 9e767d411d..b81266530b 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java @@ -15,9 +15,13 @@ */ package org.streampipes.sdk.builder.adapter; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.adapter.AdapterDescription; import org.streampipes.sdk.builder.AbstractConfigurablePipelineElementBuilder; +import java.util.Arrays; +import java.util.stream.Collectors; + public abstract class AdapterDescriptionBuilder, T extends AdapterDescription> extends AbstractConfigurablePipelineElementBuilder { @@ -27,4 +31,13 @@ protected AdapterDescriptionBuilder(String id, String label, String description, super(id, label, description, adapterTypeInstance); this.elementDescription.setAdapterId(id); } + + public AdapterDescriptionBuilder category(AdapterType... categories) { + this.elementDescription + .setCategory(Arrays + .stream(categories) + .map(Enum::name) + .collect(Collectors.toList())); + return me(); + } } diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/GuessSchemaBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/GuessSchemaBuilder.java new file mode 100644 index 0000000000..b93c38766c --- /dev/null +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/GuessSchemaBuilder.java @@ -0,0 +1,70 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.sdk.builder.adapter; + +import org.streampipes.model.connect.guess.DomainPropertyProbabilityList; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; + +import java.util.ArrayList; +import java.util.List; + +public class GuessSchemaBuilder { + + private List eventProperties; + private List domainPropertyProbabilitiesList; + + private GuessSchemaBuilder() { + this.eventProperties = new ArrayList<>(); + this.domainPropertyProbabilitiesList = new ArrayList<>(); + } + + /** + * Creates a new guess schema object using the builder pattern. + */ + public static GuessSchemaBuilder create() { + return new GuessSchemaBuilder(); + } + + public GuessSchemaBuilder property(EventProperty property) { + this.eventProperties.add(property); + + return this; + } + + public GuessSchemaBuilder domainPropertyProbability(DomainPropertyProbabilityList domainPropertyProbabilityList) { + this.domainPropertyProbabilitiesList.add(domainPropertyProbabilityList); + + return this; + } + + public GuessSchema build() { + GuessSchema guessSchema = new GuessSchema(); + EventSchema eventSchema = new EventSchema(); + + for (int i = 0; i < eventProperties.size(); i++) { + eventProperties.get(i).setIndex(i); + } + + eventSchema.setEventProperties(eventProperties); + + guessSchema.setEventSchema(eventSchema); + guessSchema.setPropertyProbabilityList(domainPropertyProbabilitiesList); + + return guessSchema; + } +} diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/ProtocolDescriptionBuilder.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/ProtocolDescriptionBuilder.java index 9adb34dbe4..a1f350589a 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/ProtocolDescriptionBuilder.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/builder/adapter/ProtocolDescriptionBuilder.java @@ -15,10 +15,14 @@ */ package org.streampipes.sdk.builder.adapter; +import org.streampipes.model.AdapterType; import org.streampipes.model.connect.grounding.ProtocolDescription; import org.streampipes.sdk.builder.AbstractConfigurablePipelineElementBuilder; import org.streampipes.sdk.helpers.AdapterSourceType; +import java.util.Arrays; +import java.util.stream.Collectors; + public class ProtocolDescriptionBuilder extends AbstractConfigurablePipelineElementBuilder { @@ -41,6 +45,15 @@ public ProtocolDescriptionBuilder sourceType(AdapterSourceType sourceType) { return this; } + public ProtocolDescriptionBuilder category(AdapterType... categories) { + this.elementDescription + .setCategory(Arrays + .stream(categories) + .map(Enum::name) + .collect(Collectors.toList())); + return me(); + } + @Override protected ProtocolDescriptionBuilder me() { return this; diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Labels.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Labels.java index 1ca60d69d7..ba91c65668 100644 --- a/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Labels.java +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Labels.java @@ -39,11 +39,20 @@ public class Labels { * @param label A human-readable title * @param description A human-readable brief summary of the element. * @return + * @deprecated Externalize labels by using + * {@link org.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * to ease future support for multiple languages. */ public static Label from(String internalId, String label, String description) { return new Label(internalId, label, description); } + @Deprecated + /** + * @deprecated Externalize labels by using + * {@link org.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * to ease future support for multiple languages. + */ public static Label fromResources(String resourceIdentifier, String resourceName) { try { return new Label(resourceName, findTitleLabel(resourceIdentifier, resourceName), findDescriptionLabel(resourceIdentifier, resourceName)); @@ -63,6 +72,12 @@ public static Label withId(String internalId) { return new Label(internalId, "", ""); } + @Deprecated + /** + * @deprecated Externalize labels by using + * {@link org.streampipes.sdk.builder.AbstractProcessingElementBuilder#withLocales(Locales...)} + * to ease future support for multiple languages. + */ public static Label withTitle(String label, String description) { return new Label("", label, description); } @@ -80,7 +95,7 @@ private static String findDescriptionLabel(String resourceIdentifier, String res } private static String makeResourceId(String resourceName, Boolean titleType) { - return resourceName +"." +(titleType ? "title" : "description"); + return resourceName + "." + (titleType ? "title" : "description"); } private static Properties loadProperties(String filename) throws IOException { diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Locales.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Locales.java new file mode 100644 index 0000000000..5a43b4ab70 --- /dev/null +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/helpers/Locales.java @@ -0,0 +1,34 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.sdk.helpers; + +public enum Locales { + + EN("en"), + DE("de"); + + private final String Filename = "strings"; + + private String code; + + Locales(String code) { + this.code = code; + } + + public String toFilename() { + return this.Filename + "." + code; + } +} diff --git a/streampipes-sdk/src/main/java/org/streampipes/sdk/utils/Assets.java b/streampipes-sdk/src/main/java/org/streampipes/sdk/utils/Assets.java new file mode 100644 index 0000000000..fa9f8fc9a1 --- /dev/null +++ b/streampipes-sdk/src/main/java/org/streampipes/sdk/utils/Assets.java @@ -0,0 +1,22 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.sdk.utils; + +public class Assets { + + public static final String DOCUMENTATION = "documentation.md"; + public static final String ICON = "icon.png"; +} diff --git a/streampipes-serializers/pom.xml b/streampipes-serializers/pom.xml index ea18a5ef18..d595863992 100644 --- a/streampipes-serializers/pom.xml +++ b/streampipes-serializers/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-serializers/src/main/java/org/streampipes/serializers/json/AdapterTypeAdapter.java b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/AdapterTypeAdapter.java new file mode 100644 index 0000000000..245cee109b --- /dev/null +++ b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/AdapterTypeAdapter.java @@ -0,0 +1,42 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.serializers.json; + +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import org.streampipes.model.AdapterType; + +import java.io.IOException; + +public class AdapterTypeAdapter extends PeTypeAdapter { + + @Override + public void write(JsonWriter out, AdapterType value) throws IOException { + write(out, value.getLabel(), value.getDescription(), value.name()); + } + + @Override + public AdapterType read(JsonReader in) throws IOException { + in.beginObject(); + while (in.hasNext()) { + String name = in.nextString(); + if (name.equals("type")) { + return AdapterType.valueOf(name); + } + } + throw new IOException(); + } +} diff --git a/streampipes-serializers/src/main/java/org/streampipes/serializers/json/EcTypeAdapter.java b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/EcTypeAdapter.java index 15a74d2f01..9b304258ea 100644 --- a/streampipes-serializers/src/main/java/org/streampipes/serializers/json/EcTypeAdapter.java +++ b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/EcTypeAdapter.java @@ -25,22 +25,20 @@ public class EcTypeAdapter extends PeTypeAdapter { - @Override - public void write(JsonWriter out, DataSinkType value) throws IOException { - write(out, value.getLabel(), value.getDescription(), value.name()); - } - - @Override - public DataSinkType read(JsonReader in) throws IOException { - in.beginObject(); - while(in.hasNext()) { - String name = in.nextString(); - if (name.equals("type")) - return DataSinkType.valueOf(name); - } - throw new IOException(); - } - - - + @Override + public void write(JsonWriter out, DataSinkType value) throws IOException { + write(out, value.getLabel(), value.getDescription(), value.name()); + } + + @Override + public DataSinkType read(JsonReader in) throws IOException { + in.beginObject(); + while (in.hasNext()) { + String name = in.nextString(); + if (name.equals("type")) { + return DataSinkType.valueOf(name); + } + } + throw new IOException(); + } } diff --git a/streampipes-serializers/src/main/java/org/streampipes/serializers/json/GsonSerializer.java b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/GsonSerializer.java index fefef7c50a..1bb9dd5ef8 100644 --- a/streampipes-serializers/src/main/java/org/streampipes/serializers/json/GsonSerializer.java +++ b/streampipes-serializers/src/main/java/org/streampipes/serializers/json/GsonSerializer.java @@ -21,6 +21,7 @@ import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.streampipes.model.AdapterType; import org.streampipes.model.DataProcessorType; import org.streampipes.model.DataSinkType; import org.streampipes.model.SpDataSet; @@ -78,6 +79,7 @@ public static GsonBuilder getGsonBuilder() { builder.registerTypeAdapter(MappingProperty.class, new JsonLdSerializer()); builder.registerTypeAdapter(ValueSpecification.class, new JsonLdSerializer()); builder.registerTypeAdapter(DataSinkType.class, new EcTypeAdapter()); + builder.registerTypeAdapter(AdapterType.class, new AdapterTypeAdapter()); builder.registerTypeAdapter(Message.class, new JsonLdSerializer()); builder.registerTypeAdapter(DataProcessorType.class, new EpaTypeAdapter()); builder.registerTypeAdapter(URI.class, new UriSerializer()); diff --git a/streampipes-sources/pom.xml b/streampipes-sources/pom.xml index 65b41988c1..126208c556 100644 --- a/streampipes-sources/pom.xml +++ b/streampipes-sources/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-sources diff --git a/streampipes-storage-api/pom.xml b/streampipes-storage-api/pom.xml index ce0883658f..226434154c 100644 --- a/streampipes-storage-api/pom.xml +++ b/streampipes-storage-api/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-storage-api jar diff --git a/streampipes-storage-api/src/main/java/org/streampipes/storage/api/IAssetDashboardStorage.java b/streampipes-storage-api/src/main/java/org/streampipes/storage/api/IAssetDashboardStorage.java new file mode 100644 index 0000000000..7f3259e38f --- /dev/null +++ b/streampipes-storage-api/src/main/java/org/streampipes/storage/api/IAssetDashboardStorage.java @@ -0,0 +1,31 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.storage.api; + +import org.streampipes.model.client.assetdashboard.AssetDashboardConfig; + +import java.util.List; + +public interface IAssetDashboardStorage { + + List getAllAssetDashboards(); + + AssetDashboardConfig getAssetDashboard(String dashboardId); + + void storeAssetDashboard(AssetDashboardConfig assetDashboardConfig); + + void deleteAssetDashboard(String dashboardId); +} diff --git a/streampipes-storage-api/src/main/java/org/streampipes/storage/api/INoSqlStorage.java b/streampipes-storage-api/src/main/java/org/streampipes/storage/api/INoSqlStorage.java index e00de2516f..23fae6a203 100644 --- a/streampipes-storage-api/src/main/java/org/streampipes/storage/api/INoSqlStorage.java +++ b/streampipes-storage-api/src/main/java/org/streampipes/storage/api/INoSqlStorage.java @@ -32,4 +32,6 @@ public interface INoSqlStorage { IVisualizationStorage getVisualizationStorageApi(); IRdfEndpointStorage getRdfEndpointStorage(); + + IAssetDashboardStorage getAssetDashboardStorage(); } diff --git a/streampipes-storage-couchdb/pom.xml b/streampipes-storage-couchdb/pom.xml index fdb418c726..379710a207 100644 --- a/streampipes-storage-couchdb/pom.xml +++ b/streampipes-storage-couchdb/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/CouchDbStorageManager.java b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/CouchDbStorageManager.java index d50a705dcb..4f64dbaa00 100644 --- a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/CouchDbStorageManager.java +++ b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/CouchDbStorageManager.java @@ -16,6 +16,7 @@ */ package org.streampipes.storage.couchdb; +import org.streampipes.storage.api.IAssetDashboardStorage; import org.streampipes.storage.api.INoSqlStorage; import org.streampipes.storage.api.INotificationStorage; import org.streampipes.storage.api.IPipelineCategoryStorage; @@ -25,6 +26,7 @@ import org.streampipes.storage.api.IRdfEndpointStorage; import org.streampipes.storage.api.IUserStorage; import org.streampipes.storage.api.IVisualizationStorage; +import org.streampipes.storage.couchdb.impl.AssetDashboardStorageImpl; import org.streampipes.storage.couchdb.impl.ConnectionStorageImpl; import org.streampipes.storage.couchdb.impl.MonitoringDataStorageImpl; import org.streampipes.storage.couchdb.impl.NotificationStorageImpl; @@ -77,4 +79,9 @@ public IVisualizationStorage getVisualizationStorageApi() { public IRdfEndpointStorage getRdfEndpointStorage() { return new RdfEndpointStorageImpl(); } + + @Override + public IAssetDashboardStorage getAssetDashboardStorage() { + return new AssetDashboardStorageImpl(); + } } diff --git a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AdapterTemplateStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AdapterTemplateStorageImpl.java index 0d32ac1d79..b6b2978952 100644 --- a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AdapterTemplateStorageImpl.java +++ b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AdapterTemplateStorageImpl.java @@ -17,7 +17,6 @@ package org.streampipes.storage.couchdb.impl; -import org.lightcouch.CouchDbClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.streampipes.model.connect.adapter.AdapterDescription; @@ -32,39 +31,39 @@ public class AdapterTemplateStorageImpl extends AbstractDao implements IAdapterTemplateStorage { - Logger LOG = LoggerFactory.getLogger(AdapterStorageImpl.class); + Logger LOG = LoggerFactory.getLogger(AdapterStorageImpl.class); - private static final String SYSTEM_USER = "system"; + private static final String SYSTEM_USER = "system"; - public AdapterTemplateStorageImpl() { - super(Utils::getCouchDbAdapterTemplateClient, AdapterDescription.class); - } + public AdapterTemplateStorageImpl() { + super(Utils::getCouchDbAdapterTemplateClient, AdapterDescription.class); + } - @Override - public List getAllAdapterTemplates() { - return findAll(); - } + @Override + public List getAllAdapterTemplates() { + return findAll(); + } - @Override - public void storeAdapterTemplate(AdapterDescription adapter) { - persist(adapter); - } + @Override + public void storeAdapterTemplate(AdapterDescription adapter) { + persist(adapter); + } - @Override - public void updateAdapterTemplate(AdapterDescription adapter) { - couchDbClientSupplier.get(). - update(adapter); - } + @Override + public void updateAdapterTemplate(AdapterDescription adapter) { + couchDbClientSupplier.get(). + update(adapter); + } - @Override - public AdapterDescription getAdapterTemplate(String adapterId) { - DbCommand, AdapterDescription> cmd = new FindCommand<>(couchDbClientSupplier, adapterId, AdapterDescription.class); - return cmd.execute().get(); - } + @Override + public AdapterDescription getAdapterTemplate(String adapterId) { + DbCommand, AdapterDescription> cmd = new FindCommand<>(couchDbClientSupplier, adapterId, AdapterDescription.class); + return cmd.execute().get(); + } - @Override - public void deleteAdapterTemplate(String adapterId) { - AdapterDescription adapterDescription = getAdapterTemplate(adapterId); - couchDbClientSupplier.get().remove(adapterDescription.getId(), adapterDescription.getRev()); - } + @Override + public void deleteAdapterTemplate(String adapterId) { + AdapterDescription adapterDescription = getAdapterTemplate(adapterId); + couchDbClientSupplier.get().remove(adapterDescription.getId(), adapterDescription.getRev()); + } } diff --git a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AssetDashboardStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AssetDashboardStorageImpl.java new file mode 100644 index 0000000000..9d0a0f2f6c --- /dev/null +++ b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/impl/AssetDashboardStorageImpl.java @@ -0,0 +1,50 @@ +/* +Copyright 2019 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.storage.couchdb.impl; + +import org.streampipes.model.client.assetdashboard.AssetDashboardConfig; +import org.streampipes.storage.api.IAssetDashboardStorage; +import org.streampipes.storage.couchdb.dao.AbstractDao; +import org.streampipes.storage.couchdb.utils.Utils; + +import java.util.List; + +public class AssetDashboardStorageImpl extends AbstractDao implements IAssetDashboardStorage { + + public AssetDashboardStorageImpl() { + super(Utils::getCouchDbAssetDashboardClient, AssetDashboardConfig.class); + } + + @Override + public List getAllAssetDashboards() { + return findAll(); + } + + @Override + public AssetDashboardConfig getAssetDashboard(String dashboardId) { + return findWithNullIfEmpty(dashboardId); + } + + @Override + public void storeAssetDashboard(AssetDashboardConfig assetDashboardConfig) { + persist(assetDashboardConfig); + } + + @Override + public void deleteAssetDashboard(String dashboardId) { + delete(dashboardId); + } +} diff --git a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/utils/Utils.java b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/utils/Utils.java index 47dbd50585..e1abb2a167 100644 --- a/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/utils/Utils.java +++ b/streampipes-storage-couchdb/src/main/java/org/streampipes/storage/couchdb/utils/Utils.java @@ -17,109 +17,113 @@ package org.streampipes.storage.couchdb.utils; -import org.streampipes.serializers.json.GsonSerializer; import org.lightcouch.CouchDbClient; import org.lightcouch.CouchDbProperties; -import org.streampipes.storage.couchdb.utils.CouchDbConfig; +import org.streampipes.serializers.json.GsonSerializer; public class Utils { - public static CouchDbClient getCouchDbAdapterTemplateClient() { - CouchDbClient dbClient = new CouchDbClient(props("adaptertemplate")); - dbClient.setGsonBuilder(GsonSerializer.getAdapterGsonBuilder()); - return dbClient; - } - - public static CouchDbClient getCouchDbAdapterClient() { - CouchDbClient dbClient = new CouchDbClient(props("adapter")); - dbClient.setGsonBuilder(GsonSerializer.getAdapterGsonBuilder()); - return dbClient; - } - - public static CouchDbClient getCouchDbPipelineClient() { - CouchDbClient dbClient = new CouchDbClient(props("pipeline")); - dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); - return dbClient; - } - - public static CouchDbClient getCouchDbSepaInvocationClient() { - CouchDbClient dbClient = new CouchDbClient(props("invocation")); - dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); - return dbClient; - } - - public static CouchDbClient getCouchDbConnectionClient() { - CouchDbClient dbClient = new CouchDbClient(props("connection")); - return dbClient; - } - - public static CouchDbClient getCouchDbVisualizationClient() { - CouchDbClient dbClient = new CouchDbClient(props("visualizations")); - return dbClient; - } - - //TODO: Remove?? - public static CouchDbClient getCouchDbRdfEndpointClient() { - CouchDbClient dbClient = new CouchDbClient(props("rdfendpoint")); - return dbClient; - } - - public static CouchDbClient getCouchDbVisualizablePipelineClient() { - CouchDbClient dbClient = new CouchDbClient(props("visualizablepipeline")); - return dbClient; - } - - public static CouchDbClient getCouchDbDashboardClient() { - CouchDbClient dbClient = new CouchDbClient(props("dashboard")); - return dbClient; - } - - public static CouchDbClient getCouchDbUserClient() { - CouchDbClient dbClient = new CouchDbClient(props("users")); - dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); - return dbClient; - } - - public static CouchDbClient getCouchDbInternalUsersClient() { - CouchDbClient dbClient = new CouchDbClient(props("_users")); - return dbClient; - } - - public static CouchDbClient getCouchDbReplicatorClient() { - CouchDbClient dbClient = new CouchDbClient(props("_replicator")); - return dbClient; - } - - public static CouchDbClient getCouchDbGlobalChangesClient() { - CouchDbClient dbClient = new CouchDbClient(props("_global_changes")); - return dbClient; - } - - - public static CouchDbClient getCouchDbMonitoringClient() { - CouchDbClient dbClient = new CouchDbClient(props("monitoring")); - return dbClient; - } - - public static CouchDbClient getCouchDbNotificationClient() { - return new CouchDbClient(props("notification")); - } - - public static CouchDbClient getCouchDbPipelineCategoriesClient() { - return new CouchDbClient(props("pipelinecategories")); - } - - public static CouchDbClient getCouchDbElasticsearchFilesEndppointClient() { - return new CouchDbClient(props("file-export-endpoints-elasticsearch")); - } - - public static CouchDbClient getCoucbDbClient(String table) { - return new CouchDbClient(props(table)); - } - - private static CouchDbProperties props(String dbname) - { - return new CouchDbProperties(dbname, true, CouchDbConfig.INSTANCE.getProtocol(), - CouchDbConfig.INSTANCE.getHost(), CouchDbConfig.INSTANCE.getPort(), null, null); - } + public static CouchDbClient getCouchDbAdapterTemplateClient() { + CouchDbClient dbClient = new CouchDbClient(props("adaptertemplate")); + dbClient.setGsonBuilder(GsonSerializer.getAdapterGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbAssetDashboardClient() { + CouchDbClient dbClient = new CouchDbClient(props("assetdashboard")); + dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbAdapterClient() { + CouchDbClient dbClient = new CouchDbClient(props("adapter")); + dbClient.setGsonBuilder(GsonSerializer.getAdapterGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbPipelineClient() { + CouchDbClient dbClient = new CouchDbClient(props("pipeline")); + dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbSepaInvocationClient() { + CouchDbClient dbClient = new CouchDbClient(props("invocation")); + dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbConnectionClient() { + CouchDbClient dbClient = new CouchDbClient(props("connection")); + return dbClient; + } + + public static CouchDbClient getCouchDbVisualizationClient() { + CouchDbClient dbClient = new CouchDbClient(props("visualizations")); + return dbClient; + } + + //TODO: Remove?? + public static CouchDbClient getCouchDbRdfEndpointClient() { + CouchDbClient dbClient = new CouchDbClient(props("rdfendpoint")); + return dbClient; + } + + public static CouchDbClient getCouchDbVisualizablePipelineClient() { + CouchDbClient dbClient = new CouchDbClient(props("visualizablepipeline")); + return dbClient; + } + + public static CouchDbClient getCouchDbDashboardClient() { + CouchDbClient dbClient = new CouchDbClient(props("dashboard")); + return dbClient; + } + + public static CouchDbClient getCouchDbUserClient() { + CouchDbClient dbClient = new CouchDbClient(props("users")); + dbClient.setGsonBuilder(GsonSerializer.getGsonBuilder()); + return dbClient; + } + + public static CouchDbClient getCouchDbInternalUsersClient() { + CouchDbClient dbClient = new CouchDbClient(props("_users")); + return dbClient; + } + + public static CouchDbClient getCouchDbReplicatorClient() { + CouchDbClient dbClient = new CouchDbClient(props("_replicator")); + return dbClient; + } + + public static CouchDbClient getCouchDbGlobalChangesClient() { + CouchDbClient dbClient = new CouchDbClient(props("_global_changes")); + return dbClient; + } + + + public static CouchDbClient getCouchDbMonitoringClient() { + CouchDbClient dbClient = new CouchDbClient(props("monitoring")); + return dbClient; + } + + public static CouchDbClient getCouchDbNotificationClient() { + return new CouchDbClient(props("notification")); + } + + public static CouchDbClient getCouchDbPipelineCategoriesClient() { + return new CouchDbClient(props("pipelinecategories")); + } + + public static CouchDbClient getCouchDbElasticsearchFilesEndppointClient() { + return new CouchDbClient(props("file-export-endpoints-elasticsearch")); + } + + public static CouchDbClient getCoucbDbClient(String table) { + return new CouchDbClient(props(table)); + } + + private static CouchDbProperties props(String dbname) { + return new CouchDbProperties(dbname, true, CouchDbConfig.INSTANCE.getProtocol(), + CouchDbConfig.INSTANCE.getHost(), CouchDbConfig.INSTANCE.getPort(), null, null); + } } diff --git a/streampipes-storage-management/pom.xml b/streampipes-storage-management/pom.xml index 333b9fadf5..4980864448 100644 --- a/streampipes-storage-management/pom.xml +++ b/streampipes-storage-management/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-storage-rdf4j/pom.xml b/streampipes-storage-rdf4j/pom.xml index fc4cd55b7d..39e1a232e4 100644 --- a/streampipes-storage-rdf4j/pom.xml +++ b/streampipes-storage-rdf4j/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-storage-rdf4j/src/main/java/org/streampipes/storage/rdf4j/impl/InMemoryStorage.java b/streampipes-storage-rdf4j/src/main/java/org/streampipes/storage/rdf4j/impl/InMemoryStorage.java index d454392b48..cdd7382021 100644 --- a/streampipes-storage-rdf4j/src/main/java/org/streampipes/storage/rdf4j/impl/InMemoryStorage.java +++ b/streampipes-storage-rdf4j/src/main/java/org/streampipes/storage/rdf4j/impl/InMemoryStorage.java @@ -25,12 +25,14 @@ import org.streampipes.model.graph.DataProcessorDescription; import org.streampipes.model.graph.DataSinkDescription; import org.streampipes.model.graph.DataSourceDescription; +import org.streampipes.model.schema.EventProperty; import org.streampipes.model.staticproperty.StaticProperty; import org.streampipes.storage.api.IPipelineElementDescriptionStorage; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -86,15 +88,21 @@ private void initializeSEPAStorage() { private void initializeSEPStorage() { inMemorySEPStorage.clear(); List seps = sesameStorage.getAllSEPs(); + seps.forEach(sep -> + sep.getSpDataStreams().forEach(es -> + es.getEventSchema() + .getEventProperties() + .sort(Comparator.comparingInt(EventProperty::getIndex)))); seps.forEach(sep -> inMemorySEPStorage.put(sep.getElementId(), sep)); seps.forEach(sep -> sep.getSpDataStreams().forEach(eventStream -> inMemoryEventStreamStorage.put(eventStream.getElementId(), eventStream))); } private List sort(List processingElements) { - processingElements.forEach(pe -> pe.getStaticProperties().sort((o1, o2) -> { - return Integer.compare(o1.getIndex(), o2.getIndex()); - })); + processingElements.forEach(pe -> { + pe.getStaticProperties().sort(Comparator.comparingInt(StaticProperty::getIndex)); + pe.getSpDataStreams().sort(Comparator.comparingInt(SpDataStream::getIndex)); + }); return processingElements; } diff --git a/streampipes-test-utils/pom.xml b/streampipes-test-utils/pom.xml index b88afa0e1c..f119d78dea 100644 --- a/streampipes-test-utils/pom.xml +++ b/streampipes-test-utils/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-user-management/pom.xml b/streampipes-user-management/pom.xml index d2307bb865..b1ed8d027c 100644 --- a/streampipes-user-management/pom.xml +++ b/streampipes-user-management/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-vocabulary/pom.xml b/streampipes-vocabulary/pom.xml index f1f26c563f..77a0ab9475 100644 --- a/streampipes-vocabulary/pom.xml +++ b/streampipes-vocabulary/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-vocabulary/src/main/java/org/streampipes/vocabulary/StreamPipes.java b/streampipes-vocabulary/src/main/java/org/streampipes/vocabulary/StreamPipes.java index e53b93f642..681785f3fe 100644 --- a/streampipes-vocabulary/src/main/java/org/streampipes/vocabulary/StreamPipes.java +++ b/streampipes-vocabulary/src/main/java/org/streampipes/vocabulary/StreamPipes.java @@ -136,6 +136,7 @@ public class StreamPipes { public static final String HAS_OUTPUT_STRATEGY = NS + "hasOutputStrategy"; public static final String HAS_EPA_TYPE = NS + "hasEpaType"; public static final String HAS_EC_TYPE = NS + "hasEcType"; + public static final String HAS_ADAPTER_TYPE = NS + "hasAdapterType"; public static final String PRODUCES = NS + "produces"; public static final String HAS_TRANSPORT_PROTOCOL = NS + "hasTransportProtocol"; @@ -321,4 +322,7 @@ public class StreamPipes { public static final String SUCCESS_MESSAGE = NS + "SuccessMessage"; + public static final String INCLUDED_ASSETS = NS + "includedAssets"; + public static final String INCLUDES_LOCALES = NS + "includesLocales"; + public static final String INCLUDED_LOCALES = NS + "includedLocales"; } diff --git a/streampipes-wrapper-distributed/pom.xml b/streampipes-wrapper-distributed/pom.xml index 0a6395af35..28bc23e7d3 100644 --- a/streampipes-wrapper-distributed/pom.xml +++ b/streampipes-wrapper-distributed/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-wrapper-esper/pom.xml b/streampipes-wrapper-esper/pom.xml index 5d0b2400c3..b593ef1266 100644 --- a/streampipes-wrapper-esper/pom.xml +++ b/streampipes-wrapper-esper/pom.xml @@ -3,7 +3,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-wrapper-flink/pom.xml b/streampipes-wrapper-flink/pom.xml index d9935550f1..b7c4ae3481 100644 --- a/streampipes-wrapper-flink/pom.xml +++ b/streampipes-wrapper-flink/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-wrapper-flink StreamPipes Wrapper for Apache Flink diff --git a/streampipes-wrapper-kafka-streams/pom.xml b/streampipes-wrapper-kafka-streams/pom.xml index c8a83d7725..4ccf43bd15 100644 --- a/streampipes-wrapper-kafka-streams/pom.xml +++ b/streampipes-wrapper-kafka-streams/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 diff --git a/streampipes-wrapper-siddhi/pom.xml b/streampipes-wrapper-siddhi/pom.xml index 11cbd438c1..eeb611c020 100644 --- a/streampipes-wrapper-siddhi/pom.xml +++ b/streampipes-wrapper-siddhi/pom.xml @@ -20,14 +20,14 @@ streampipes-parent org.streampipes - 0.61.0 + 0.62.0 4.0.0 streampipes-wrapper-siddhi - 4.1.51 + 4.5.4 diff --git a/streampipes-wrapper-siddhi/src/main/java/org/streampipes/wrapper/siddhi/engine/SiddhiEventEngine.java b/streampipes-wrapper-siddhi/src/main/java/org/streampipes/wrapper/siddhi/engine/SiddhiEventEngine.java index c228c3da1f..3dc8bdeb40 100644 --- a/streampipes-wrapper-siddhi/src/main/java/org/streampipes/wrapper/siddhi/engine/SiddhiEventEngine.java +++ b/streampipes-wrapper-siddhi/src/main/java/org/streampipes/wrapper/siddhi/engine/SiddhiEventEngine.java @@ -18,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.streampipes.model.graph.DataProcessorInvocation; import org.streampipes.model.runtime.EventFactory; import org.streampipes.model.runtime.SchemaInfo; import org.streampipes.model.runtime.SourceInfo; @@ -32,11 +33,7 @@ import org.wso2.siddhi.core.stream.input.InputHandler; import org.wso2.siddhi.core.stream.output.StreamCallback; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.StringJoiner; +import java.util.*; public abstract class SiddhiEventEngine implements EventProcessor { @@ -47,12 +44,15 @@ public abstract class SiddhiEventEngine i private Map siddhiInputHandlers; private List inputStreamNames; + private List sortedEventKeys; + private static final Logger LOG = LoggerFactory.getLogger(SiddhiEventEngine.class); public SiddhiEventEngine() { this.siddhiAppString = new StringBuilder(); this.siddhiInputHandlers = new HashMap<>(); this.inputStreamNames = new ArrayList<>(); + sortedEventKeys = new ArrayList<>(); } @Override @@ -63,11 +63,12 @@ public void onInvocation(B parameters, SpOutputCollector spOutputCollector, Even SiddhiManager siddhiManager = SpSiddhiManager.INSTANCE.getSiddhiManager(); + LOG.info("Configuring event types for graph " + parameters.getGraph().getName()); parameters.getInEventTypes().forEach((key, value) -> { - Map inTypeMap = value; - registerEventTypeIfNotExists(key, inTypeMap); - this.inputStreamNames.add(fixEventName(key)); + // TODO why is the prefix not in the parameters.getInEventType + registerEventTypeIfNotExists( key, value); + this.inputStreamNames.add(prepareName(key)); }); String fromStatement = fromStatement(inputStreamNames, parameters); @@ -77,15 +78,15 @@ public void onInvocation(B parameters, SpOutputCollector spOutputCollector, Even siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiAppString.toString()); parameters .getInEventTypes() - .forEach((key, value) -> siddhiInputHandlers.put(key, siddhiAppRuntime.getInputHandler(fixEventName(key)))); + .forEach((key, value) -> siddhiInputHandlers.put(key, siddhiAppRuntime.getInputHandler(prepareName(key)))); - siddhiAppRuntime.addCallback(fixEventName(getOutputTopicName(parameters)), new StreamCallback() { + siddhiAppRuntime.addCallback(prepareName(getOutputTopicName(parameters)), new StreamCallback() { @Override public void receive(Event[] events) { for(Event event : events) { // TODO provide collector in RuntimeContext - spOutputCollector.collect(toSpEvent(event, parameters, runtimeContext.getOutputSchemaInfo - (), runtimeContext.getOutputSourceInfo())); + spOutputCollector.collect(toSpEvent(event, parameters, runtimeContext.getOutputSchemaInfo + (), runtimeContext.getOutputSourceInfo())); } } }); @@ -104,24 +105,26 @@ private String getOutputTopicName(B parameters) { private org.streampipes.model.runtime.Event toSpEvent(Event event, B parameters, SchemaInfo schemaInfo, - SourceInfo sourceInfo) { + SourceInfo sourceInfo) { Map outMap = new HashMap<>(); - int i = 0; - // TODO make sure that ordering of event attributes is correct - for (String key : parameters.getOutEventType().keySet()) { - outMap.put(key, event.getData(i)); - i++; + for (int i = 0; i < sortedEventKeys.size(); i++) { + outMap.put(sortedEventKeys.get(i), event.getData(i)); } return EventFactory.fromMap(outMap, sourceInfo, schemaInfo); } private void registerEventTypeIfNotExists(String eventTypeName, Map typeMap) { - String defineStreamPrefix = "define stream " + fixEventName(eventTypeName); + String defineStreamPrefix = "define stream " + prepareName(eventTypeName); StringJoiner joiner = new StringJoiner(","); for (String key : typeMap.keySet()) { - joiner.add(key + " " + toType((Class) typeMap.get(key))); + sortedEventKeys.add(key); + Collections.sort(sortedEventKeys); + }; + + for (String key : sortedEventKeys) { + joiner.add("s0" + key + " " + toType((Class) typeMap.get(key))); } this.siddhiAppString.append(defineStreamPrefix); @@ -131,7 +134,6 @@ private void registerEventTypeIfNotExists(String eventTypeName, Map o) { - System.out.println(o.getCanonicalName()); if (o.equals(Long.class)) { return "LONG"; } else if (o.equals(Integer.class)) { @@ -139,7 +141,7 @@ private String toType(Class o) { } else if (o.equals(Double.class)) { return "DOUBLE"; } else if (o.equals(Float.class)) { - return "FLOAT"; + return "DOUBLE"; } else if (o.equals(Boolean.class)) { return "BOOL"; } else { @@ -148,15 +150,15 @@ private String toType(Class o) { } private void registerStatements(String fromStatement, String selectStatement, String outputStream) { - this.siddhiAppString.append(fromStatement) - .append("\n") - .append(selectStatement) - .append("\n") - .append("insert into ") - .append(fixEventName(outputStream)) - .append(";"); + this.siddhiAppString.append(fromStatement) + .append("\n") + .append(selectStatement) + .append("\n") + .append("insert into ") + .append(prepareName(outputStream)) + .append(";"); - LOG.info("Registering statement: \n" +this.siddhiAppString.toString()); + LOG.info("Registering statement: \n" +this.siddhiAppString.toString()); } @@ -170,7 +172,12 @@ public void onEvent(org.streampipes.model.runtime.Event event, SpOutputCollector } private Object[] toObjArr(Map event) { - return event.values().toArray(); + Object[] result = new Object[sortedEventKeys.size()]; + for (int i = 0; i < sortedEventKeys.size(); i++) { + result[i] = event.get(sortedEventKeys.get(i)); + } + + return result; } @Override @@ -181,7 +188,31 @@ public void onDetach() { protected abstract String fromStatement(List inputStreamNames, final B bindingParameters); protected abstract String selectStatement(final B bindingParameters); - private String fixEventName(String eventName) { - return eventName.replaceAll("\\.", ""); + protected String prepareName(String eventName) { + return eventName + .replaceAll("\\.", "") + .replaceAll("-", "") + .replaceAll("::", ""); + } + + + protected String getCustomOutputSelectStatement(DataProcessorInvocation invocation) { + StringBuilder selectString = new StringBuilder(); + selectString.append("select "); + + if (sortedEventKeys.size() > 0) { + for (int i = 0; i < sortedEventKeys.size() -1; i++) { + selectString.append("e1.s0" + sortedEventKeys.get(i) + ","); + + } + selectString.append("e1.s0" + sortedEventKeys.get(sortedEventKeys.size() - 1)); + + } + + return selectString.toString(); + } + + public void setSortedEventKeys(List sortedEventKeys) { + this.sortedEventKeys = sortedEventKeys; } } \ No newline at end of file diff --git a/streampipes-wrapper-spark/pom.xml b/streampipes-wrapper-spark/pom.xml index 390955b2a5..fb627d0765 100644 --- a/streampipes-wrapper-spark/pom.xml +++ b/streampipes-wrapper-spark/pom.xml @@ -22,7 +22,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-wrapper-spark diff --git a/streampipes-wrapper-standalone/pom.xml b/streampipes-wrapper-standalone/pom.xml index f6a56ade91..91deb678cc 100644 --- a/streampipes-wrapper-standalone/pom.xml +++ b/streampipes-wrapper-standalone/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-wrapper-standalone StreamPipes Wrapper for Standalone Pipeline Element Implementations diff --git a/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpCollector.java b/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpCollector.java index bcb4a97050..17816b7e23 100644 --- a/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpCollector.java +++ b/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpCollector.java @@ -25,8 +25,8 @@ import org.streampipes.wrapper.routing.PipelineElementCollector; import org.streampipes.wrapper.standalone.manager.PManager; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public abstract class StandaloneSpCollector implements PipelineElementCollector { @@ -47,7 +47,7 @@ public StandaloneSpCollector(T protocol, TransportFormat format) throws SpRuntim this.transportFormat = format; this.dataFormatDefinition = PManager.getDataFormat(format).orElseThrow(() -> new SpRuntimeException("Could not find format")); - this.consumers = new HashMap<>(); + this.consumers = new ConcurrentHashMap<>(); } public void registerConsumer(String routeId, C consumer) { diff --git a/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpInputCollector.java b/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpInputCollector.java index 76361fec32..586b080270 100644 --- a/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpInputCollector.java +++ b/streampipes-wrapper-standalone/src/main/java/org/streampipes/wrapper/standalone/routing/StandaloneSpInputCollector.java @@ -44,9 +44,7 @@ public void onEvent(byte[] event) { if (singletonEngine) { send(consumers.get(consumers.keySet().toArray()[0]), event); } else { - consumers.keySet().forEach(c -> { - send(consumers.get(c), event); - }); + consumers.forEach((key, value) -> send(value, event)); } } diff --git a/streampipes-wrapper/pom.xml b/streampipes-wrapper/pom.xml index 8a7725e2ea..9291838091 100644 --- a/streampipes-wrapper/pom.xml +++ b/streampipes-wrapper/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.61.0 + 0.62.0 streampipes-wrapper