diff --git a/haikudepotserver-docs/pom.xml b/haikudepotserver-docs/pom.xml index 32d7a065..63ace375 100644 --- a/haikudepotserver-docs/pom.xml +++ b/haikudepotserver-docs/pom.xml @@ -1,7 +1,12 @@ + + - + haikudepotserver-parent org.haiku ../haikudepotserver-parent diff --git a/pom.xml b/pom.xml index a7c92d38..849c5c14 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,9 @@ + + 4.0.0 @@ -20,7 +25,6 @@ - rpm-capable-build @@ -38,6 +42,7 @@ + diff --git a/support/deployment/Dockerfile b/support/deployment/Dockerfile new file mode 100644 index 00000000..62692783 --- /dev/null +++ b/support/deployment/Dockerfile @@ -0,0 +1,61 @@ +# ===================================== +# Copyright 2018, Andrew Lindesay +# Distributed under the terms of the MIT License. +# ===================================== + +# This Dockerfile is used to build an image that is able to run an HDS server. +# Do not use this file from the master or other working branch, but instead +# use one from a release tag because this version of the file will have the +# correct versions inserted into it. See the documentation for further +# details on how this deployment mechanism works. + +FROM ubuntu:16.04 + +# These environment variables will be set using the release script. It will +# copy the correct values from the poms into these variables so that the right +# versions of the artefacts are used in the Docker container. + +ENV HDS_VERSION "???" +ENV JETTY_VERSION "???" +ENV PG_VERSION "???" + +ENV JAVA_BIN "java" + +ENV INSTALL_ROOT "/opt/haikudepotserver" +ENV INSTALL_HVIF2PNG_PATH "${INSTALL_ROOT}/hvif2png-hrev51165/bin/hvif2png.sh" + +ENV HDS_WAR "haikudepotserver-webapp-${HDS_VERSION}.war" +ENV HDS_WAR_DOWNLOAD_URL "https://github.com/aplgithub/haikudepotserver/releases/download/haikudepotserver-${HDS_VERSION}/${HDS_WAR}" +ENV HDS_PORT 8080 + +ENV JETTY_JAR "jetty-runner-${JETTY_VERSION}.jar" +ENV JETTY_DOWNLOAD_URL "http://central.maven.org/maven2/org/eclipse/jetty/jetty-runner/${JETTY_VERSION}/${JETTY_JAR}" + +ENV PG_JAR "postgresql-${PG_VERSION}.jar" +ENV PG_DOWNLOAD_URL "http://central.maven.org/maven2/org/postgresql/postgresql/${PG_VERSION}/${PG_JAR}" + +ENV HVIF2PNG_VERSION "hvif2png-hrev51165-linux-x86_64" + +RUN mkdir -p "${INSTALL_ROOT}" +RUN apt-get update && apt-get -y install optipng libpng12-0 curl openjdk-8-jdk fontconfig ttf-dejavu + +COPY config.properties ${INSTALL_ROOT} +COPY logback.xml ${INSTALL_ROOT} +COPY launch.sh ${INSTALL_ROOT} + +ADD ${JETTY_DOWNLOAD_URL} ${INSTALL_ROOT}/${JETTY_JAR} +ADD ${HVIF2PNG_VERSION}.tgz ${INSTALL_ROOT} +ADD ${PG_DOWNLOAD_URL} ${INSTALL_ROOT}/${PG_JAR} +ADD ${HDS_WAR_DOWNLOAD_URL} ${INSTALL_ROOT}/${HDS_WAR} + +RUN echo "HDS_ROOT=${INSTALL_ROOT}" > ${INSTALL_ROOT}/launchenv.sh +RUN echo "JAVA_BIN=${JAVA_BIN}" >> ${INSTALL_ROOT}/launchenv.sh +RUN echo "HDS_HVIF2PNG_PATH=${INSTALL_HVIF2PNG_PATH}" >> ${INSTALL_ROOT}/launchenv.sh +RUN echo "HDS_PORT=${HDS_PORT}" >> ${INSTALL_ROOT}/launchenv.sh +RUN echo "HDS_WAR=${HDS_WAR}" >> ${INSTALL_ROOT}/launchenv.sh +RUN echo "JETTY_JAR=${JETTY_JAR}" >> ${INSTALL_ROOT}/launchenv.sh +RUN echo "PG_JAR=${PG_JAR}" >> ${INSTALL_ROOT}/launchenv.sh + +CMD [ "sh", "/opt/haikudepotserver/launch.sh" ] + +EXPOSE ${HDS_PORT} diff --git a/support/deployment/config.properties b/support/deployment/config.properties new file mode 100644 index 00000000..b1881889 --- /dev/null +++ b/support/deployment/config.properties @@ -0,0 +1,128 @@ +# +# Copyright 2018, Andrew Lindesay +# Distributed under the terms of the MIT License. +# + +# =========================================== +# haikudepotserver docker - configuration file +# =========================================== + +# ------------------------------------------- +# general + +captcha.expiryseconds=240 + +pkgversion.viewcounter.protectrecurringincrementfromsameclient=true + +deployment.isproduction=true + +architecture.default.code=x86_gcc2 + +optipng.path=/usr/bin/optipng + +# When set (either "true" or "false"), the repository import process will +# obtain the data for the package and will thereby figure out the size of +# the package. +repository.import.populatepayloadlength=true + +# Configures a minimum version for the HaikiDepot desktop application. +# Versions of HD desktop application less than this minimum are +# disallowed. + +desktop.application.version.min=0.0.1 + +# If configured, this path should point to the "hvif2png.sh" script. See +# the documentation on information as to where this can be sourced from. + +# docker env-vars +#hvif2png.path= + + +# ------------------------------------------- +# database connection + +# docker env-vars +jdbc.driver=org.postgresql.Driver +#jdbc.url= +#jdbc.username= +#jdbc.password= + +flyway.migrate=true + +# This should be true in production. +flyway.validateOnMigrate=false + +# When configured, this value will indicate the size of the cache dedicated to storing +# query results. It will default to a sensible value. + +#cayenne.query.cache.size=250 + +# ------------------------------------------- +# user ratings + +# How many versions to go back from the latest version in order +# to find user ratings to aggregate. + +userrating.aggregation.pkg.versionsback=2 + +# How many ratings must be present before an aggregate can be +# derived; too few and the result will be from too small a sample +# to be meaningful. + +userrating.aggregation.pkg.minratings=1 + +# ------------------------------------------- +# web infrastructure + +# This value relates to the built-in JAWR web-resource compression and +# concatination system. Review the JAWR documentation for details on +# the effects of configuring this value to true. + +#jawr.debug.on=false + +# This URL provides a base URL that the application can then add to when +# it formulates URLs that are to be used outside of the application; for +# example, URLs in ATOM feeds etc... + +baseurl=https://depot.haiku-os.org + +# ------------------------------------------- +# web security + +# This is the number of seconds for which an authentication token is +# valid for. + +#authentication.jws.expiryseconds= + +# This secret is used to sign the tokens used to communicate between +# the client and the server. This value should be very hard to guess. +# The output of the command "uuidgen" would be appropriate. This +# value must be kept secret and not disclosed in public. +# commented out to force the value to be considered + +#authentication.jws.sharedkey= + +# This value is used in the production and consumption of the tokens +# between the client and the server. It is intended that this is +# consistent for a given deployment. Sample values that may be +# appropriate; +# +# dev.hds +# prod.hds +# test.hds +# + +authentication.jws.issuer=prod.hds + +# ------------------------------------------- +# email-related + +#smtp.host=smtp +#smtp.port=25 +#smtp.username= +#smtp.password= +#smtp.auth=false +#smtp.starttls=false +email.from=noreply@haiku-os.org + +# ------------------------------------------- \ No newline at end of file diff --git a/support/deployment/hds_secrets b/support/deployment/hds_secrets new file mode 100644 index 00000000..9af30d13 --- /dev/null +++ b/support/deployment/hds_secrets @@ -0,0 +1,9 @@ +# This is an example secrets file that might be used to convey +# system secrets to the HDS application through a shared directory +# to the docker container. + +HDS_JDBC_URL=jdbc:postgresql://somepghost:5432/haikudepotserver +HDS_JDBC_USERNAME=somepguser +HDS_JDBC_PASSWORD=somepgpassword +HDS_SMTP_HOST=somesmtpserver +HDS_AUTHENTICATION_JWS_ISSUER=justtesting.hds diff --git a/support/deployment/hvif2png-hrev51165-linux-x86_64.tgz b/support/deployment/hvif2png-hrev51165-linux-x86_64.tgz new file mode 100644 index 00000000..40e601b4 Binary files /dev/null and b/support/deployment/hvif2png-hrev51165-linux-x86_64.tgz differ diff --git a/support/deployment/launch.sh b/support/deployment/launch.sh new file mode 100644 index 00000000..05148d82 --- /dev/null +++ b/support/deployment/launch.sh @@ -0,0 +1,25 @@ +# launch file for the haikudepotserver system. + +. "$(dirname $0)/launchenv.sh" +. "/secrets/hds_secrets" + +"${JAVA_BIN}" \ +"-Dfile.encoding=UTF-8" \ +"-Dlogback.configurationFile=${DS_ROOT}/logback.xml" \ +"-Duser.timezone=GMT0" \ +"-Xms256m" \ +"-Xmx320m" \ +"-Djava.net.preferIPv4Stack=true" \ +"-Djava.awt.headless=true" \ +"-Dconfig.properties=file://${HDS_ROOT}/config.properties" \ +"-Dhvif2png.path=${HDS_HVIF2PNG_PATH}" \ +"-Djdbc.url=${HDS_JDBC_URL}" \ +"-Djdbc.username=${HDS_JDBC_USERNAME}" \ +"-Djdbc.password=${HDS_JDBC_PASSWORD}" \ +"-Dsmtp.host=${HDS_SMTP_HOST}" \ +"-Dauthentication.jws.issuer=${HDS_AUTHENTICATION_JWS_ISSUER}" \ +"-jar" "${HDS_ROOT}/${JETTY_JAR}" \ +"--jar" "${HDS_ROOT}/${PG_JAR}" \ +"--port" "${HDS_PORT}" \ +"${HDS_ROOT}/${HDS_WAR}" \ + diff --git a/support/deployment/logback.xml b/support/deployment/logback.xml new file mode 100644 index 00000000..5886bd82 --- /dev/null +++ b/support/deployment/logback.xml @@ -0,0 +1,23 @@ + + + + + + + + %d{ISO8601} %X{authUserNickname}/%X{userAgentCode} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/support/hdscommon.py b/support/hdscommon.py index 402ff84e..e4703fb1 100644 --- a/support/hdscommon.py +++ b/support/hdscommon.py @@ -1,6 +1,6 @@ # ===================================== -# Copyright 2014-2015, Andrew Lindesay +# Copyright 2014-2018, Andrew Lindesay # Distributed under the terms of the MIT License. # ===================================== @@ -12,6 +12,7 @@ import os import os.path import sys +import re import xml.etree.ElementTree as etree import subprocess @@ -49,13 +50,13 @@ def scanmodules(): def extractdefaultnamespace(tag): if "{" != tag[0]: - print "invalid tag missing namespace (open) ; " + tag + print("invalid tag missing namespace (open) ; " + tag) sys.exit(1) closebraceindex = tag.find("}") if -1 == closebraceindex: - print "invalid tag missing namespace (close) ; " + tag + print("invalid tag missing namespace (close) ; " + tag) sys.exit(1) return tag[1:closebraceindex] @@ -68,7 +69,7 @@ def pomtoplevelelement(tree, taglocalname): el = roote.find("{"+namespace+"}"+taglocalname) if el is None: - print "unable to find the "+taglocalname+" element" + print("unable to find the "+taglocalname+" element") sys.exit(1) return el @@ -85,6 +86,24 @@ def pomextractversion(tree): return pomtoplevelelement(tree, "version").text +def pomextractproperty(tree, name): + propertiese = pomtoplevelelement(tree, "properties") + namespace = extractdefaultnamespace(propertiese.tag) + propertye = propertiese.find("{"+namespace+"}" + name) + + if propertye is None: + raise Exception('unable to find the property [' + name + ']') + + print(propertye) + + property = propertye.text + + if not property or 0 == len(property): + raise Exception('no stored value for property [' + name + ']') + + return property + + # ===================================== # LOGIC CHUNKS @@ -92,7 +111,7 @@ def ensurecurrentversionconsistencyformodule(modulename, expectedversion): modulepomtree = etree.parse(modulename + "/pom.xml") if not modulepomtree: - print "the 'pom.xml' for module "+modulename+" should be accessible" + print("the 'pom.xml' for module "+modulename+" should be accessible") sys.exit(1) parente = pomtoplevelelement(modulepomtree, "parent") @@ -100,31 +119,75 @@ def ensurecurrentversionconsistencyformodule(modulename, expectedversion): versione = parente.find("{"+namespace+"}version") if versione is None: - print "the parent element of module " + modulename + " has no version specified" + print("the parent element of module " + modulename + " has no version specified") sys.exit(1) else: actualversion = versione.text if actualversion != expectedversion: - print "the version of the module "+modulename+" is inconsistent with the expected; " + actualversion + print("the version of the module "+modulename+" is inconsistent with the expected; " + actualversion) sys.exit(1) else: - print modulename + ": " + actualversion + " (ok)" + print(modulename + ": " + actualversion + " (ok)") # ===================================== # GIT + +def gitcommitversion(version): + if 0 == subprocess.call(["git", "commit", "-m", "version " + version]): + print("git committed 'version " + version + "'") + else: + raise RuntimeError("failed to git commit [" + version + "]") + + +def gitaddfile(file): + if 0 == subprocess.call(["git", "add", file]): + print(file + " : (added)") + else: + raise RuntimeError("failed to git-add [" + file + "]") + + def gitaddpomformodule(modulename): if modulename is None: - if 0 == subprocess.call(["git", "add", "pom.xml"]): - print "pom.xml: (added)" - else: - print "failed to git-add; pom.xml" - sys.exit(1) + gitaddfile("pom.xml") else: - if 0 == subprocess.call(["git", "add", modulename + "/pom.xml"]): - print modulename + "/pom.xml: (added)" - else: - print("failed to git-add; "+modulename+"/pom.xml") - sys.exit(1) + gitaddfile(modulename + "/pom.xml") + + +# ===================================== +# MAVEN + + +def mvnversionsset(version): + if 0 == subprocess.call(["mvn", "-q", "versions:set", "-DnewVersion=" + version, "-DgenerateBackupPoms=false"]): + print("versions:set to " + version) + else: + raise RuntimeError("failed to set maven versions to [" + version + "]") + + +# ===================================== +# DOCKERFILE + + +def dockerreplaceenvs(filename, replacements): + with open(filename, "r") as dockerfile: + dockerlines = dockerfile.readlines() + + def maybereplaceenv(l): + envmatch = re.match("^ENV ([A-Z0-9_]+) \".+\"$", l) + + if envmatch: + envname = envmatch.group(1) + replacementvalue = replacements.get(envname) + + if replacementvalue: + return 'ENV ' + envname + ' "' + replacementvalue + '"\n' + + return l + + dockerlines = list(map(maybereplaceenv, dockerlines)) + + with open(filename, "w") as dockerfile: + dockerfile.writelines(dockerlines) diff --git a/support/hdsrelease.py b/support/hdsrelease.py index 6bf0abea..d9e0a370 100644 --- a/support/hdsrelease.py +++ b/support/hdsrelease.py @@ -1,7 +1,5 @@ -#!/usr/bin/python - # ===================================== -# Copyright 2014-2017, Andrew Lindesay +# Copyright 2014-2018, Andrew Lindesay # Distributed under the terms of the MIT License. # ===================================== @@ -16,21 +14,35 @@ import subprocess +DOCKERFILE = 'support/deployment/Dockerfile' + + +def gitaddpomanddockerfiles(): + print("will git-add pom files") + hdscommon.gitaddpomformodule(None) + + for m in rootPomModuleNames: + hdscommon.gitaddpomformodule(m) + + print('will git-add Dockerfile') + hdscommon.gitaddfile(DOCKERFILE) + + # ---------------- # PARSE TOP-LEVEL POM AND GET MODULE NAMES if not os.path.isfile("pom.xml"): - print "the 'pom.xml' file should be accessible in the present working directory" + print("the 'pom.xml' file should be accessible in the present working directory") sys.exit(1) rootPomTree = etree.parse("pom.xml") if not rootPomTree: - print "the 'pom.xml' should be accessible in the present working directory" + print("the 'pom.xml' should be accessible in the present working directory") sys.exit(1) if hdscommon.pomextractartifactid(rootPomTree) != "haikudepotserver": - print "the top level pom should have the 'haikudepotserver' artifactId" + print("the top level pom should have the 'haikudepotserver' artifactId") sys.exit(1) rootPomModuleNames = hdscommon.scanmodules() @@ -42,13 +54,13 @@ rootPomCurrentVersionMatch = re.match("^([1-9][0-9]*\.[0-9]+\.)([1-9][0-9]*)-SNAPSHOT$", rootPomCurrentVersion) if not rootPomCurrentVersionMatch: - print "the current root pom version is not a valid snapshot version; " + rootPomCurrentVersion + print("the current root pom version is not a valid snapshot version; " + rootPomCurrentVersion) sys.exit(1) rootPomCurrentVersionPrefix = rootPomCurrentVersionMatch.group(1) rootPomCurrentVersionSuffix = rootPomCurrentVersionMatch.group(2) -print "top-level version; " + rootPomCurrentVersion +print("top-level version; " + rootPomCurrentVersion) releaseVersion = rootPomCurrentVersionPrefix + rootPomCurrentVersionSuffix futureVersion = rootPomCurrentVersionPrefix + str(int(rootPomCurrentVersionSuffix) + 1) + "-SNAPSHOT" @@ -58,7 +70,7 @@ # This will make sure that all of the modules have the same version. -print "will check version consistency" +print("will check version consistency") for m in rootPomModuleNames: hdscommon.ensurecurrentversionconsistencyformodule(m, rootPomCurrentVersion) @@ -66,59 +78,61 @@ # ---------------- # RESET THE VERSIONS SANS THE SNAPSHOT -if 0 == subprocess.call(["mvn", "-q", "versions:set", "-DnewVersion=" + releaseVersion, "-DgenerateBackupPoms=false"]): - print "versions:set to "+releaseVersion -else: - print "failed version:set to " + releaseVersion - sys.exit(1) +hdscommon.mvnversionsset(releaseVersion) # ---------------- -# ADD POMS TO GIT, COMMIT AND TAG +# SETUP THE DOCKERFILE WITH THE RIGHT VERSIONS -print "will git-add pom files" -hdscommon.gitaddpomformodule(None) -for m in rootPomModuleNames: - hdscommon.gitaddpomformodule(m) +def replacedockerenvs(): + parentpome = etree.parse("haikudepotserver-parent/pom.xml") + postgresversion = hdscommon.pomextractproperty(parentpome, 'postgresql.version') + jettyversion = hdscommon.pomextractproperty(parentpome, 'jetty.version') + envreplacements = { + 'HDS_VERSION': releaseVersion, + 'JETTY_VERSION': jettyversion, + 'PG_VERSION': postgresversion + } + hdscommon.dockerreplaceenvs(DOCKERFILE, envreplacements) -if 0 == subprocess.call(["git", "commit", "-m", "version " + releaseVersion]): - print "git committed 'version " + releaseVersion + "'" -else: - print "failed to git commit" - sys.exit(1) + +replacedockerenvs() + + +# ---------------- +# ADD POMS TO GIT, COMMIT AND TAG + +gitaddpomanddockerfiles() +hdscommon.gitcommitversion(releaseVersion) if 0 == subprocess.call(["git", "tag", "-a", "haikudepotserver-" + releaseVersion, "-m", "haikudepotserver-" + releaseVersion]): - print "git tagged 'haikudepotserver-" + releaseVersion + "'" + print("git tagged 'haikudepotserver-" + releaseVersion + "'") else: - print "failed to git tag" + print("failed to git tag") sys.exit(1) # ---------------- # UPDATE ALL POMS TO NEW SNAPSHOT -if 0 == subprocess.call(["mvn", "-q", "versions:set", "-DnewVersion=" + futureVersion, "-DgenerateBackupPoms=false"]): - print "versions:set to "+futureVersion -else: - print "failed version:set to " + futureVersion - sys.exit(1) +hdscommon.mvnversionsset(futureVersion) # ---------------- -# ADD POMS TO GIT, COMMIT +# RESET THE DOCKERFILE TO PLACEBO VALUE -print "will git-add pom files" -hdscommon.gitaddpomformodule(None) +hdscommon.dockerreplaceenvs(DOCKERFILE, { + 'HDS_VERSION': '???', + 'JETTY_VERSION': '???', + 'PG_VERSION': '???' +}) -for m in rootPomModuleNames: - hdscommon.gitaddpomformodule(m) +# ---------------- +# ADD POMS TO GIT, COMMIT -if 0 == subprocess.call(["git", "commit", "-m", "version " + futureVersion]): - print "git committed 'version " + futureVersion + "'" -else: - print "failed to git commit" - sys.exit(1) +gitaddpomanddockerfiles() +hdscommon.gitcommitversion(futureVersion) # ---------------- # REMINDER AT THE END TO PUSH -print "---------------" -print "to complete the release; git push && git push --tags" \ No newline at end of file +print("---------------") +print("to complete the release; git push && git push --tags") \ No newline at end of file