diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0f7511f --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +# Eclipse +*.pydevproject +.project +.metadata +bin/** +tmp/** +tmp/**/* +*.tmp +*.bak +*.swp +*~.nib +local.properties +.classpath +.settings/ +.loadpath +# Emacs +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +.elc +auto-save-list +tramp +# IntelliJ Idea +*.iml +*.ipr +*.iws +.idea/ +# Maven +target/ +# NetBeans +nbproject/ +# Vim +.*.sw[a-z] +*.un~ +Session.vim +# Ensime +.ensime +.ensime_cache/ +.ensime*.log +# Windows +Thumbs.db +Desktop.ini diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..f1e1f47 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,16 @@ +# Only format files tracked by git. +project.git = true +# Default style with align. +style = defaultWithAlign +# Other options... +danglingParentheses = true +maxColumn = 100 +project.excludeFilters = [".*\\.sbt"] +rewrite.rules = [RedundantBraces, RedundantParens, SortImports] +rewriteTokens = { + "=>" = "⇒" + "<-" = "←" + "->" = "→" +} +spaces.inImportCurlyBraces = true +unindentTopLevelOperators = true diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..14c0ff3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: scala +scala: + - 2.11.11 +jdk: + - oraclejdk8 +script: + - sbt clean coverage test coverageReport +#after_success: +# - bash <(curl -s https://codecov.io/bash) + diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..5b6b6f6 --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,12 @@ +Corporate Contributors +====================== + +- Copyright (c) 2014 - 2015 Wegtam UG (haftungsbeschränkt) +- Copyright (c) 2015 - 2017 Wegtam GmbH + +Individual Contributors +======================= + +- Jens Grassel +- André Schütz + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4d10318 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,362 @@ +# ChangeLog for the Tensei-Data API + +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](http://semver.org/). + +## Conventions when editing this file. + +Please follow the listed conventions when editing this file: + +* one subsection per version +* reverse chronological order (latest entry on top) +* write all dates in iso notation (`YYYY-MM-DD`) +* each version should group changes according to their impact: + * `Added` for new features. + * `Changed` for changes in existing functionality. + * `Deprecated` for once-stable features removed in upcoming releases. + * `Removed` for deprecated features removed in this release. + * `Fixed` for any bug fixes. + * `Security` to invite users to upgrade in case of vulnerabilities. + +## Unreleased + +## 1.92.0 (2017-08-25) + +### Changed + +- update sbt to 0.13.16 and serveral plugins +- switch akka dependency to `provided` + +### Removed + +- `BaseApplication` and `Settings` because they are no longer used. +- Akka is no longer bundled in the publised artefact + +## 1.91.0 (2017-05-18) + +### Added + +- protobuf messages + +### Canged + +- use default secure random number generator from JVM + +### Fixed + +- strong secure random number generator blocks endless on systems with low entropy + +## 1.90.0 (2017-03-09) + +### Added + +- add field `languageTag` to `ConnectionInformation` + +## 1.89.0 (2017-03-06) + +### Added + +- ParserState extended + +## 1.88.0 (2017-03-03) + +### Removed + +- unused/obsolete classes + +## 1.87.0 (2017-03-03) + +### Added + +- custom class for schema extractor options + +### Changed + +- update Akka to 2.4.17 +- global message `ExtractSchema` now uses `ExtractSchemaOptions` instead of strings +- switch to scalafmt for code formatting +- more strict compiler flags + +## 1.86.0 (2016-06-23) + +### Added + +- files for contribution guide + - [AUTHORS.md](AUTHORS.md) + - this CHANGELOG file + - [CONTRIBUTING.md](CONTRIBUTING.md) + - [LICENSE](LICENSE) + +### Changed + +- update Akka to 2.4.7 + +### Fixed + +- minor code style issues + +## 1.85.0 (2016-05-10) + +### Added + +- activate `sbt-wartremover` + +### Changed + +- code cleanup +- update Akka to 2.3.15 + +### Removed + +- `-Xfatal-warnings` flag +- unused dependencies + +## 1.84.0 (2016-03-18) + +### Changed + +- add agent id to log message classes + +## 1.83.0 (2016-03-16) + +### Added + +- classes for log message handling +- `sbt-wartremover` for clean code warnings + +### Changed + +- major code cleanup +- sbt plugin configuration + +## 1.82.1 (2016-03-14) + +### Added + +- compiler flags for better Java 8 support + - `-Ybackend:GenBCode` + - `-Ydelambdafy:method` + +### Changed + +- update Scala to 2.11.8 +- update sbt to 0.13.11 + +## 1.82.0 (2016-02-22) + +### Removed + +- `TenseiDataType` because it is only used in the agent + +## 1.81.0 (2016-02-19) + +### Added + +- data type for `Boolean` + +## 1.80.0 (2016-02-18) + +### Added + +- helper function to convert data types into `TenseiDataType` (boxing) + +## 1.79.0 (2016-02-16) + +### Added + +- container for binary data + +## 1.78.0 (2016-02-16) + +### Changed + +- renamed trait `TenseiDataTypes` to `TenseiDataType` + +## 1.77.0 (2016-02-16) + +### Added + +- data type for `ElementReference` +- data type for auto increment values +- `sbt-scalariform` for code style enforcement + +### Changed + +- restructure sbt plugins +- update scalatest and switch test matchers from `should` to `must` +- clean up sbt resolver settings + +### Removed + +- `sbt-scoverage` because of hard linking of profiling dependencies + +## 1.73.0 (2015-10-13) + +### Changed + +- update Akka to 2.3.14 +- update sbt-scoverage +- update `.gitignore` +- update sbt to 0.13.9 +- code cleanup for scalaz disjunction + +## 1.72.0 (2015-07-17) + +### Added + +- messages for statistics + +## 1.71.0 (2015-07-15) + +### Changed + +- update Akka to 2.3.12 +- prefer scala version defined in `build.sbt` upon conflicts + +## 1.70.1 (2015-06-30) + +### Changed + +- add field for group id to `User` +- adjust permissions checks to group names which have to be unique +- maximum value in `License` is now `Int.MaxValue` instead of `-1` +- publish to our own repository (Apache Archiva) +- update Akka to 2.3.11 +- update Scala to 2.11.7 +- extend messages for schema extraction + +## 1.64.0 (2015-05-04) + +### Added + +- message `NoLicenseInstalled` + +## 1.63.1 (2015-04-27) + +### Added + +- move some messages from agent into the api +- move some messages from server into the api +- helper method on `Cookbook` that returns mapped source ids +- special messages for license handling +- helpers for cryptographic functions +- helper method to create licenses +- code coverage via `sbt-scoverage` + +### Changed + +- update Scala to 2.11.6 +- renames and refactoring +- `Recipe` can include an empty mapping list +- replace transformation id with unique identifier (UUID) + +### Fixed + +- some typos + +## 1.36.0 (2014-12-30) + +### Changed + +- new agent state `CleaningUp` + +## 1.35.0 (2014-12-30) + +### Changed + +- remove `Bootable` from base cluster application + +### Removed + +- remove Akka microkernel + +## 1.34.0 (2014-12-29) + +### Added + +- integrated akka-camel +- messages for push notification support from server to frontend + +### Changed + +- update Akka to 2.3.8 +- more fields for runtime statistics + +## 1.31.0 (2014-12-15) + +### Added + +- included Akka microkernel and `Bootable` +- global messages to create a `DFASDL` from a `ConnectionInformation` +- global messages for errors + +### Changed + +- base application extends `Bootable` +- new agent state `Aborting` + +## 1.28.0 (2014-12-10) + +### Added + +- sealed trait and tests for states of agent, processor and parser +- implement `equals` and `hashCode` on `DFASDL` + +### Changed + +- update sbt to 0.13.7 +- require test run before `sbt publish` + +### Fixed + +- error in tests + +## 1.21.1 (2014-11-13) + +### Added + +- version field for `DFASDL` +- role name for cluster nodes +- base class for cluster applications +- `DFASDL` function for generating missing element ids +- global messages +- license data type +- agent working states +- runtime statistics + +### Changed + +- update Akka to 2.3.7 +- migration to Java 8 and Scala 2.11 +- `target` field in `Cookbook` is now optional +- `DFASDL` checks for empty id +- `ConnectionInformation` fields use `Option` now instead of empty strings + +## 1.2.1 (2014-07-11) + +### Changed + +- adjust json codec for key field definition + +## 1.2.0 (2014-07-11) + +### Changed + +- adjust mapping key field definition + +## 1.1.0 (2014-07-11) + +### Added + +- key field definitions + +## 1.0.1 (2014-07-10) + +### Added + +- Tests +- helper functions + +## 1.0.0 (2014-07-10) + +- initial release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3a6f45b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,44 @@ +# Contribution Guide + +This project has adopted the [Collective Code Construction Contract +(C4.2)](https://rfc.zeromq.org/spec:42) for contributing. Please read it +before sending patches. + +Everyone is expected to follow the +[Scala Code of Conduct](http://www.scala-lang.org/conduct.html) when +dicussing the project on the available communication channels. +If you are being harassed, please contact us immediately so that we can +support you. + +### Additions to C4.2 + +1. This project is licensed under GNU Affero General Public License 3. +See [LICENSE](LICENSE) for details. + +2. Contributors are listed in the file [AUTHORS.md](AUTHORS.md). Add +yourself if you have contributed. + +3. Please maintain the existing code style and try to keep your commits +small and focused. The project follows the at large the +[Scala Style Guide](http://docs.scala-lang.org/style/). Code is formatted +using [scalafmt](http://scalameta.org/scalafmt/) upon compilation. You +may also use a scalafmt plugin for your favourite editor/IDE. + +4. Please rebase your branch if the project diverges from your branch. + +5. Before a pull request is merged the commits done on the feature branch +SHOULD be squashed into a single commit. + +6. Changes are documented in the file [CHANGELOG.md](CHANGELOG.md). Please +use the section `Unreleased` to note your changes. + +## Release Guide + +The changes in the section `Unreleased` in the [CHANGELOG.md](CHANGELOG.md) +file MUST be moved to a section named after the release and a new empty +`Unreleased` section MUST be created. + +A release SHALL be accompanied by an annotated tag (`git tag -a NAME`) that +holds a description of the changes that are included in the release. This +description SHOULD be same as in the file [CHANGELOG.md](CHANGELOG.md). + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..dba13ed --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..bffda00 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Tensei API + +[![Build Status](https://travis-ci.org/Tensei-Data/tensei-api.svg?branch=master)](https://travis-ci.org/Tensei-Data/tensei-api) +[![codecov](https://codecov.io/gh/Tensei-Data/tensei-api/branch/master/graph/badge.svg)](https://codecov.io/gh/Tensei-Data/tensei-api) +[![Download](https://api.bintray.com/packages/wegtam/tensei-data/tensei-api/images/download.svg)](https://bintray.com/wegtam/tensei-data/tensei-api/_latestVersion) + +This project is intended to provide libraries that provide shared +functionality between several Tensei-Data components. + +Currently the following sub projects are included in this repository: + +1. generic tensei-api library +2. protobuf messages for remoting +3. benchmarks + +## Resources + +The main website for Tensei-Data is located at: https://www.wegtam.com/products/tensei-data + +### Mailing lists + +[![Google-Group tensei-data](https://img.shields.io/badge/group-tensei--data-brightgreen.svg)](https://groups.google.com/forum/#!forum/tensei-data) +[![Google-Group tensei-data-dev](https://img.shields.io/badge/group-tensei--data--dev-orange.svg)](https://groups.google.com/forum/#!forum/tensei-data-dev) diff --git a/benchmarks/src/jmh/scala/com/wegtam/tensei/MessageSerializerBenchmark.scala b/benchmarks/src/jmh/scala/com/wegtam/tensei/MessageSerializerBenchmark.scala new file mode 100644 index 0000000..02f5700 --- /dev/null +++ b/benchmarks/src/jmh/scala/com/wegtam/tensei/MessageSerializerBenchmark.scala @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei + +import java.io.{ByteArrayOutputStream, ObjectOutputStream} +import java.net.URI + +import com.wegtam.tensei.adt._ +import com.wegtam.tensei.remote.adt.StartTransformation +import org.openjdk.jmh.annotations._ + +@State(Scope.Benchmark) +@Fork(3) +@Warmup(iterations = 4) +@Measurement(iterations = 10) +@BenchmarkMode(Array(Mode.Throughput)) +class MessageSerializerBenchmark { + + private final val dfasdl = + """ + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + """.stripMargin + + private final val protobufStartTransformationMessage = StartTransformation( + id = scala.util.Random.alphanumeric.take(32).mkString, + cookbook = Option(com.wegtam.tensei.remote.adt.Cookbook( + id = "COOKBOOK", + sources = List( + com.wegtam.tensei.remote.adt.DFASDL( + id = "DFASDL-1", + content = dfasdl + ) + ), + target = Option(com.wegtam.tensei.remote.adt.DFASDL( + id = "DFASDL-2", + content = dfasdl + )), + recipes = List( + com.wegtam.tensei.remote.adt.Recipe( + id = "ID1", + mode = com.wegtam.tensei.remote.adt.RecipeMode.RecipeMapOneToOne, + mappings = List( + com.wegtam.tensei.remote.adt.MappingTransformation( + sources = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "id"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "name"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "description"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "birthday"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "salary") + ), + targets = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "id"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "name"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "description"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "birthday"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "salary") + ), + transformations = List() + ) + ) + ), + com.wegtam.tensei.remote.adt.Recipe( + id = "ID2", + mode = com.wegtam.tensei.remote.adt.RecipeMode.RecipeMapOneToOne, + mappings = List( + com.wegtam.tensei.remote.adt.MappingTransformation( + sources = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "id2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "name2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "description2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "birthday2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "salary2") + ), + targets = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "id2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "name2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "description2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "birthday2"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "salary2") + ), + transformations = List() + ) + ) + ), + com.wegtam.tensei.remote.adt.Recipe( + id = "ID3", + mode = com.wegtam.tensei.remote.adt.RecipeMode.RecipeMapOneToOne, + mappings = List( + com.wegtam.tensei.remote.adt.MappingTransformation( + sources = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "id3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "name3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "description3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "birthday3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-1", "salary3") + ), + targets = List( + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "id3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "name3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "description3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "birthday3"), + com.wegtam.tensei.remote.adt.ElementReference("DFASDL-2", "salary3") + ), + transformations = List() + ) + ) + ) + ) + )), + sources = List( + com.wegtam.tensei.remote.adt.ConnectionInformation( + uri = "http://example.com/", + dfasdlRef = Option(com.wegtam.tensei.remote.adt.DFASDLReference( + cookbookId = "COOKBOOK", + dfasdlId = "DFASDL-1" + )) + ) + ), + target = Option(com.wegtam.tensei.remote.adt.ConnectionInformation( + uri = "http://target.example.com/", + dfasdlRef = Option(com.wegtam.tensei.remote.adt.DFASDLReference( + cookbookId = "COOKBOOK", + dfasdlId = "DFASDL-2" + )) + )) + ) + + private final val startTransformationMessage = AgentStartTransformationMessage( + sources = List( + ConnectionInformation( + uri = new URI("http://example.com/"), + dfasdlRef = Option(DFASDLReference( + cookbookId = "COOKBOOK", + dfasdlId = "DFASDL-1" + )), + username = None, + password = None, + checksum = None, + languageTag = None + ) + ), + target = ConnectionInformation( + uri = new URI("http://target.example.com/"), + dfasdlRef = Option(DFASDLReference( + cookbookId = "COOKBOOK", + dfasdlId = "DFASDL-2" + )) + ), + cookbook = Cookbook( + id = "COOKBOOK", + sources = List(DFASDL(id = "DFASDL-1", content = dfasdl)), + target = Option(DFASDL(id = "DFASDL-2", content = dfasdl)), + recipes = List( + Recipe.createOneToOneRecipe("ID1", List( + MappingTransformation( + List( + ElementReference("DFASDL-1", "id"), + ElementReference("DFASDL-1", "name"), + ElementReference("DFASDL-1", "description"), + ElementReference("DFASDL-1", "birthday"), + ElementReference("DFASDL-1", "salary") + ), + List( + ElementReference("DFASDL-2", "id"), + ElementReference("DFASDL-2", "name"), + ElementReference("DFASDL-2", "description"), + ElementReference("DFASDL-2", "birthday"), + ElementReference("DFASDL-2", "salary") + ), + List() + ) + )), + Recipe.createOneToOneRecipe("ID2", List( + MappingTransformation( + List( + ElementReference("DFASDL-1", "id2"), + ElementReference("DFASDL-1", "name2"), + ElementReference("DFASDL-1", "description2"), + ElementReference("DFASDL-1", "birthday2"), + ElementReference("DFASDL-1", "salary2") + ), + List( + ElementReference("DFASDL-2", "id2"), + ElementReference("DFASDL-2", "name2"), + ElementReference("DFASDL-2", "description2"), + ElementReference("DFASDL-2", "birthday2"), + ElementReference("DFASDL-2", "salary2") + ), + List() + ) + )), + Recipe.createOneToOneRecipe("ID3", List( + MappingTransformation( + List( + ElementReference("DFASDL-1", "id3"), + ElementReference("DFASDL-1", "name3"), + ElementReference("DFASDL-1", "description3"), + ElementReference("DFASDL-1", "birthday3"), + ElementReference("DFASDL-1", "salary3") + ), + List( + ElementReference("DFASDL-2", "id3"), + ElementReference("DFASDL-2", "name3"), + ElementReference("DFASDL-2", "description3"), + ElementReference("DFASDL-2", "birthday3"), + ElementReference("DFASDL-2", "salary3") + ), + List() + ) + )) + ) + ), + uniqueIdentifier = Option(scala.util.Random.alphanumeric.take(32).mkString) + ) + private final val dfasdlObject = DFASDL( + id = "MY-DFASDL-ID", + content = dfasdl + ) + private final val protoBufDfasdlObject = com.wegtam.tensei.remote.adt.DFASDL( + id = "MY-DFASDL-ID", + content = dfasdl + ) + + @Benchmark + def writeStartTransformationMessage: Array[Byte] = { + val bs = new ByteArrayOutputStream() + val os = new ObjectOutputStream(bs) + os.writeObject(startTransformationMessage) + os.close() + bs.toByteArray + } + + @Benchmark + def writeStartTransformationMessageProtoBuf: Array[Byte] = { + protobufStartTransformationMessage.toByteArray + } + + @Benchmark + def writeDfasdlObject: Array[Byte] = { + val bs = new ByteArrayOutputStream() + val os = new ObjectOutputStream(bs) + os.writeObject(dfasdlObject) + os.close() + bs.toByteArray + } + + @Benchmark + def writeDfasdlObjectProtoBuf: Array[Byte] = { + protoBufDfasdlObject.toByteArray + } + +} diff --git a/benchmarks/src/jmh/scala/com/wegtam/tensei/security/CryptoHelpersBenchmark.scala b/benchmarks/src/jmh/scala/com/wegtam/tensei/security/CryptoHelpersBenchmark.scala new file mode 100644 index 0000000..6852c83 --- /dev/null +++ b/benchmarks/src/jmh/scala/com/wegtam/tensei/security/CryptoHelpersBenchmark.scala @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.security + +import org.openjdk.jmh.annotations._ + +import scalaz.{-\/, \/, \/-} + +@State(Scope.Benchmark) +@Fork(3) +@Warmup(iterations = 4) +@Measurement(iterations = 10) +@BenchmarkMode(Array(Mode.Throughput)) +class CryptoHelpersBenchmark extends CryptoHelpers { + + private val (aesKey, aesIV) = generateAESKeyAndIV() + private val keyPair = generateRSAKeyPair() + private val sourceData = scala.util.Random.alphanumeric.take(4096).mkString.getBytes("UTF-8") + private val rsaSourceData = scala.util.Random.alphanumeric.take(200).mkString.getBytes("UTF-8") + private val signature = sign(sourceData, keyPair.getPrivate) match { + case -\/(failure) => throw failure + case \/-(success) => java.util.Base64.getDecoder.decode(success) + } + + private val aesData = encrypt(sourceData, getAESCipher, aesKey, Option(aesIV)) match { + case -\/(failure) => throw failure + case \/-(success) => java.util.Base64.getDecoder.decode(success) + } + private val rsaData = encrypt(rsaSourceData, getRSACipher, keyPair.getPublic, None) match { + case -\/(failure) => throw failure + case \/-(success) => java.util.Base64.getDecoder.decode(success) + } + + @Benchmark + def encrypAESBenchmark: \/[Throwable, Array[Byte]] = { + encrypt(sourceData, getAESCipher, aesKey, Option(aesIV)) + } + + @Benchmark + def encrypRSABenchmark: \/[Throwable, Array[Byte]] = { + encrypt(rsaSourceData, getRSACipher, keyPair.getPublic, None) + } + + @Benchmark + def decryptAESBenchmark: \/[Throwable, Array[Byte]] = { + decrypt(aesData, getAESCipher, aesKey, Option(aesIV)) + } + + @Benchmark + def decryptRSABenchmark: \/[Throwable, Array[Byte]] = { + decrypt(rsaData, getRSACipher, keyPair.getPrivate, None) + } + + @Benchmark + def signBenchmark: \/[Throwable, Array[Byte]] = { + sign(sourceData, keyPair.getPrivate) + } + + @Benchmark + def validateBenchmark: Boolean = { + validate(sourceData, signature, keyPair.getPublic) + } + +} diff --git a/benchmarks/src/main/scala/com/wegtam/tensei/MessageSerializerMemoryUsage.scala b/benchmarks/src/main/scala/com/wegtam/tensei/MessageSerializerMemoryUsage.scala new file mode 100644 index 0000000..f199b49 --- /dev/null +++ b/benchmarks/src/main/scala/com/wegtam/tensei/MessageSerializerMemoryUsage.scala @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei + +import java.io.{ ByteArrayOutputStream, ObjectOutputStream } + +import com.wegtam.tensei.adt.DFASDL +import org.github.jamm.MemoryMeter + +object MessageSerializerMemoryUsage { + + private final val dfasdl = + """ + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + """.stripMargin + private final val dfasdlObject = DFASDL( + id = "MY-DFASDL-ID", + content = dfasdl + ) + private final val protoBufDfasdlObject = com.wegtam.tensei.remote.adt.DFASDL( + id = "MY-DFASDL-ID", + content = dfasdl + ) + + def main(args: Array[String]): Unit = { + val meter = new MemoryMeter() + val classicBytes: Array[Byte] = { + val bs = new ByteArrayOutputStream() + val os = new ObjectOutputStream(bs) + os.writeObject(dfasdlObject) + os.close() + bs.toByteArray + } + val protoBufBytes = protoBufDfasdlObject.toByteArray + + val srcSize = meter.measureDeep(dfasdl) + val classicSize = meter.measureDeep(classicBytes) + val protoSize = meter.measureDeep(protoBufBytes) + + println("DFASDL Sizes:") + println(s"\tXML-String\t\t: $srcSize bytes") + println(s"\tClassic Serializer\t: $classicSize bytes") + println(s"\tProtobuf Serializer\t: $protoSize bytes") + } + +} diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..73e3064 --- /dev/null +++ b/build.sbt @@ -0,0 +1,206 @@ +// ***************************************************************************** +// Projects +// ***************************************************************************** + +// Calculate the current year for usage in copyright notices and license headers. +lazy val currentYear: Int = java.time.OffsetDateTime.now().getYear + +lazy val tenseiApi = + project + .in(file(".")) + .enablePlugins(AutomateHeaderPlugin, GitBranchPrompt, GitVersioning) + .settings(settings ++ publishSettings) + .settings( + name := "tensei-api", + libraryDependencies ++= Seq( + library.akkaActor % Provided, + library.argonaut, + library.akkaTestkit % Test, + library.scalaCheck % Test, + library.scalaTest % Test + ), + wartremoverWarnings in (Compile, compile) ++= Warts.unsafe + ) + .aggregate(tenseiApiMessages) + +lazy val tenseiApiMessages = + project + .in(file("messages")) + .enablePlugins(AutomateHeaderPlugin, GitBranchPrompt, GitVersioning) + .settings(settings ++ protoBufSettings) + .settings( + name := "tensei-api-messages", + libraryDependencies ++= Seq( + library.scalaCheck % Test, + library.scalaTest % Test, + // The following dependencies are just needed because ScalaPB bails out with an error otherwise. :-( + library.easyMock % Test, + library.easyMockClassic % Test, + library.jUnit % Test + ) + ) + +lazy val benchmarks = + project + .in(file("benchmarks")) + .enablePlugins(AutomateHeaderPlugin, GitBranchPrompt, GitVersioning, JmhPlugin) + .settings(settings) + .settings( + name := "tensei-api-benchmarks", + javaOptions ++= (dependencyClasspath in Test).map(makeAgentOptions).value, + javaOptions in run ++= List("-Xmx2g", "-XX:MaxMetaspaceSize=2g"), + fork := true, + libraryDependencies ++= Seq( + library.jaInstrumenter, + library.jaMemoryMeter + ) + ) + .dependsOn(tenseiApi, tenseiApiMessages) + +/** + * Helper function to generate options for instrumenting memory analysis. + * + * @param cp The current classpath. + * @return A list of options (strings). + */ +def makeAgentOptions(cp: Classpath): Seq[String] = { + val jammJar = cp.map(_.data).filter(_.toString.contains("jamm")).head + val jaiJar = cp.map(_.data).filter(_.toString.contains("instrumenter")).head + Seq(s"-javaagent:$jammJar", s"-javaagent:$jaiJar") +} + +// ***************************************************************************** +// Library dependencies +// ***************************************************************************** + +lazy val library = + new { + object Version { + val akka = "2.4.17" + val argonaut = "6.0.4" + val cats = "0.9.0" + val circe = "0.7.0" + val easyMock = "3.4" + val easyMockClassic = "3.2" + val jaMemoryMeter = "0.3.1" + val jaInstrumenter = "3.0.1" + val jUnit = "4.12" + val scalaCheck = "1.13.5" + val scalaTest = "3.0.4" + } + val akkaActor: ModuleID = "com.typesafe.akka" %% "akka-actor" % Version.akka + val akkaCluster: ModuleID = "com.typesafe.akka" %% "akka-cluster" % Version.akka + val akkaTestkit: ModuleID = "com.typesafe.akka" %% "akka-testkit" % Version.akka + val argonaut: ModuleID = "io.argonaut" %% "argonaut" % Version.argonaut + val cats: ModuleID = "org.typelevel" %% "cats" % Version.cats + val circeCore: ModuleID = "io.circe" %% "circe-core" % Version.circe + val circeGeneric: ModuleID = "io.circe" %% "circe-generic" % Version.circe + val circeParser: ModuleID = "io.circe" %% "circe-parser" % Version.circe + val easyMock: ModuleID = "org.easymock" % "easymock" % Version.easyMock + val easyMockClassic: ModuleID = "org.easymock" % "easymockclassextension" % Version.easyMockClassic + val jUnit: ModuleID = "junit" % "junit" % Version.jUnit + val jaMemoryMeter: ModuleID = "com.github.jbellis" % "jamm" % Version.jaMemoryMeter + val jaInstrumenter: ModuleID = "com.google.code.java-allocation-instrumenter" % "java-allocation-instrumenter" % Version.jaInstrumenter + val scalaCheck: ModuleID = "org.scalacheck" %% "scalacheck" % Version.scalaCheck + val scalaTest: ModuleID = "org.scalatest" %% "scalatest" % Version.scalaTest + } + +// ***************************************************************************** +// Settings +// ***************************************************************************** + +lazy val settings = + commonSettings ++ + gitSettings ++ + resolverSettings ++ + scalafmtSettings + +lazy val commonSettings = + Seq( + headerLicense := Some(HeaderLicense.AGPLv3(s"2014 - $currentYear", "Contributors as noted in the AUTHORS.md file")), + scalaVersion in ThisBuild := "2.11.11", + //crossScalaVersions := Seq("2.11.11", "2.12.3"), + organization := "com.wegtam.tensei", + organizationName := "Wegtam GmbH", + startYear := Option(2014), + licenses += ("AGPL-V3", url("https://www.gnu.org/licenses/agpl.html")), + mappings.in(Compile, packageBin) += baseDirectory.in(ThisBuild).value / "LICENSE" -> "LICENSE", + scalacOptions ++= Seq( + "-deprecation", + "-encoding", "UTF-8", + "-feature", + "-target:jvm-1.8", + "-unchecked", + //"-Xfatal-warnings", + "-Xfuture", + "-Xlint", + "-Ybackend:GenBCode", + "-Ydelambdafy:method", + "-Yno-adapted-args", + "-Ywarn-numeric-widen", + "-Ywarn-unused-import", + "-Ywarn-value-discard" + ), + javacOptions ++= Seq( + "-encoding", "UTF-8", + "-source", "1.8", + "-target", "1.8" + ), + unmanagedSourceDirectories.in(Compile) := Seq(scalaSource.in(Compile).value), + unmanagedSourceDirectories.in(Test) := Seq(scalaSource.in(Test).value), + incOptions := incOptions.value.withNameHashing(nameHashing = true), + autoAPIMappings := true + ) + +lazy val gitSettings = + Seq( + git.useGitDescribe := true + ) + +lazy val protoBufSettings = + Seq( + PB.runProtoc := (args => Process("protoc", args).!), + PB.targets in Compile := Seq( + scalapb.gen( + flatPackage = true + ) -> (sourceManaged in Compile).value + ), + libraryDependencies += "com.trueaccord.scalapb" %% "scalapb-runtime" % com.trueaccord.scalapb.compiler.Version.scalapbVersion % "protobuf" + ) + +lazy val publishSettings = + Seq( + bintrayOrganization := Option("wegtam"), + bintrayPackage := "tensei-api", + bintrayReleaseOnPublish in ThisBuild := false, + bintrayRepository := "tensei-data", + developers += Developer( + "wegtam", + "Wegtam GmbH", + "tech@wegtam.com", + url("https://www.wegtam.com") + ), + homepage := Option(url("https://github.com/Tensei-Data/tensei-api")), + pomIncludeRepository := (_ => false), + publishArtifact in Test := false, + publish := (publish dependsOn (test in Test)).value, + scmInfo := Option( + ScmInfo( + url("https://github.com/Tensei-Data/tensei-api"), + "git@github.com:Tensei-Data/tensei-api.git" + ) + ) + ) + +lazy val resolverSettings = + Seq( + resolvers += "Tensei-Data" at "https://dl.bintray.com/wegtam/tensei-data" + ) + +lazy val scalafmtSettings = + Seq( + scalafmtOnCompile := true, + scalafmtOnCompile.in(Sbt) := false, + scalafmtVersion := "1.2.0" + ) + diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ActorRef.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ActorRef.proto new file mode 100644 index 0000000..88ac4eb --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ActorRef.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message ActorRef { + string path = 1; // Use `Serialization.serializedActorPath(theActorRef)` to get the path and `extendedSystem.provider.resolveActorRef(identifier)` to get an ActorRef. +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AgentInformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AgentInformation.proto new file mode 100644 index 0000000..0cbad7d --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AgentInformation.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ActorRef.proto"; +import "com/wegtam/tensei/remote/states/AgentAuthorisationState.proto"; +import "com/wegtam/tensei/remote/states/AgentWorkingState.proto"; + +message AgentInformation { + string id = 1; + com.wegtam.tensei.remote.adt.ActorRef ref = 2; + com.wegtam.tensei.remote.states.AgentAuthorisationState auth = 3; + int64 last_updated = 4; + com.wegtam.tensei.remote.states.AgentWorkingState working_state = 5; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AtomicTransformationDescription.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AtomicTransformationDescription.proto new file mode 100644 index 0000000..11d80d0 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/AtomicTransformationDescription.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ElementReference.proto"; +import "com/wegtam/tensei/remote/adt/TransformationOptions.proto"; + +message AtomicTransformationDescription { + ElementReference element = 1; + string transformer_class_name = 2; + TransformationOptions options = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ConnectionInformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ConnectionInformation.proto new file mode 100644 index 0000000..f575ad7 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ConnectionInformation.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/DFASDLReference.proto"; + +message ConnectionInformation { + string uri = 1; + DFASDLReference dfasdl_ref = 2; + string username = 3; + string password = 4; + string checksum = 5; + string language_tag = 6; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Cookbook.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Cookbook.proto new file mode 100644 index 0000000..97f8248 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Cookbook.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/DFASDL.proto"; +import "com/wegtam/tensei/remote/adt/Recipe.proto"; + +message Cookbook { + string id = 1; + repeated DFASDL sources = 2; + DFASDL target = 3; + repeated Recipe recipes = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDL.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDL.proto new file mode 100644 index 0000000..f7e8b38 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDL.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message DFASDL { + string id = 1; + string content = 2; + string version = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDLReference.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDLReference.proto new file mode 100644 index 0000000..7eb5f8a --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/DFASDLReference.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message DFASDLReference { + string cookbook_id = 1; + string dfasdl_id = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ElementReference.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ElementReference.proto new file mode 100644 index 0000000..c0cccb0 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ElementReference.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message ElementReference { + string dfasdl_id = 1; + string element_id = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ExtractSchemaOptions.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ExtractSchemaOptions.proto new file mode 100644 index 0000000..fc7e994 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/ExtractSchemaOptions.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message ExtractSchemaOptions { + bool csv_header = 1; + string csv_separator = 2; + string encoding = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingKeyFieldDefinition.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingKeyFieldDefinition.proto new file mode 100644 index 0000000..a8b09d9 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingKeyFieldDefinition.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message MappingKeyFieldDefinition { + string name = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingTransformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingTransformation.proto new file mode 100644 index 0000000..f5a7e5d --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/MappingTransformation.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ElementReference.proto"; +import "com/wegtam/tensei/remote/adt/AtomicTransformationDescription.proto"; +import "com/wegtam/tensei/remote/adt/MappingKeyFieldDefinition.proto"; +import "com/wegtam/tensei/remote/adt/TransformationDescription.proto"; + +message MappingTransformation { + repeated ElementReference sources = 1; + repeated ElementReference targets = 2; + repeated TransformationDescription transformations = 3; + repeated AtomicTransformationDescription atomic_transformations = 4; + MappingKeyFieldDefinition mapping_key = 5; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Recipe.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Recipe.proto new file mode 100644 index 0000000..c65056b --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/Recipe.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/MappingTransformation.proto"; + +enum RecipeMode { + RecipeMapAllToAll = 0; + RecipeMapOneToOne = 1; +} + +message Recipe { + string id = 1; + repeated MappingTransformation mappings = 2; + RecipeMode mode = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/RuntimeStats.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/RuntimeStats.proto new file mode 100644 index 0000000..87a53b1 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/RuntimeStats.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message RuntimeStats { + string hostname = 1; + int64 free_memory = 2; + int64 max_memory = 3; + int64 total_memory = 4; + int32 processors = 5; + double system_load = 6; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TenseiLicense.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TenseiLicense.proto new file mode 100644 index 0000000..e9e4619 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TenseiLicense.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +import "scalapb/scalapb.proto"; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message TenseiLicense { + string id = 1; + string licensee = 2; + int32 agents = 3; + int32 users = 4; + int32 configurations = 5; + int32 cronjobs = 6; + int32 triggers = 7; + int64 expiration_date = 8; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationDescription.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationDescription.proto new file mode 100644 index 0000000..8225421 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationDescription.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/TransformationOptions.proto"; + +message TransformationDescription { + string transformer_class_name = 1; + TransformationOptions transformation_options = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationOptions.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationOptions.proto new file mode 100644 index 0000000..0677db2 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/adt/TransformationOptions.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message TransformationOption { + string name = 1; + string value = 2; +} + +message TransformationOptions { + repeated TransformationOption options = 1; + string src_type = 2; + string dst_type = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortTransformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortTransformation.proto new file mode 100644 index 0000000..29388a5 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortTransformation.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message AbortTransformation { + string sender = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortingTransformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortingTransformation.proto new file mode 100644 index 0000000..068a7c2 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AbortingTransformation.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message AbortingTransformation { + string sender = 1; + string message = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReportToRef.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReportToRef.proto new file mode 100644 index 0000000..6b48e9a --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReportToRef.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ActorRef.proto"; + +message AgentReportToRef { + com.wegtam.tensei.remote.adt.ActorRef ref = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReporting.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReporting.proto new file mode 100644 index 0000000..8be42e3 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentReporting.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ActorRef.proto"; + +message AgentReporting { + string id = 1; + com.wegtam.tensei.remote.adt.ActorRef ref = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogLine.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogLine.proto new file mode 100644 index 0000000..960c118 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogLine.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message AgentRunLogLine { + string run_id = 1; + string log_line = 2; + int64 offset = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogsMetaData.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogsMetaData.proto new file mode 100644 index 0000000..31aff8d --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/AgentRunLogsMetaData.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message AgentRunLogsMetaData { + string agent_id = 1; + string run_id = 2; + int64 size = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchema.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchema.proto new file mode 100644 index 0000000..ab96249 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchema.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = false; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ConnectionInformation.proto"; +import "com/wegtam/tensei/remote/adt/ExtractSchemaOptions.proto"; + +message ExtractSchema { + com.wegtam.tensei.remote.adt.ConnectionInformation source = 1; + com.wegtam.tensei.remote.adt.ExtractSchemaOptions options = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchemaResult.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchemaResult.proto new file mode 100644 index 0000000..04ad0a8 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ExtractSchemaResult.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ConnectionInformation.proto"; +import "com/wegtam/tensei/remote/adt/DFASDL.proto"; +import "com/wegtam/tensei/remote/adt/ExtractSchemaOptions.proto"; + +message ExtractSchemaResult { + com.wegtam.tensei.remote.adt.ConnectionInformation source = 1; + com.wegtam.tensei.remote.adt.ExtractSchemaOptions options = 2; + com.wegtam.tensei.remote.adt.DFASDL dfasdl = 3; + string error_message = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogs.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogs.proto new file mode 100644 index 0000000..752d096 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogs.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message RequestAgentRunLogs { + string id = 1; + string run_id = 2; + int64 offset = 3; + int64 max_size = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogsMetaData.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogsMetaData.proto new file mode 100644 index 0000000..c6cbcff --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/RequestAgentRunLogsMetaData.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message RequestAgentRunLogsMetaData { + string run_id = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Restart.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Restart.proto new file mode 100644 index 0000000..54bef6a --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Restart.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message Restart { + +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReportToRef.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReportToRef.proto new file mode 100644 index 0000000..876a817 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReportToRef.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ActorRef.proto"; + +message ServerReportToRef { + com.wegtam.tensei.remote.adt.ActorRef ref = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReporting.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReporting.proto new file mode 100644 index 0000000..d0865a3 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/ServerReporting.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ActorRef.proto"; + +message ServerReporting { + com.wegtam.tensei.remote.adt.ActorRef ref = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Shutdown.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Shutdown.proto new file mode 100644 index 0000000..11815d6 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/Shutdown.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message Shutdown { + +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StartTransformation.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StartTransformation.proto new file mode 100644 index 0000000..0189926 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StartTransformation.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.adt; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/ConnectionInformation.proto"; +import "com/wegtam/tensei/remote/adt/Cookbook.proto"; + +message StartTransformation { + string id = 1; + com.wegtam.tensei.remote.adt.Cookbook cookbook = 2; + repeated com.wegtam.tensei.remote.adt.ConnectionInformation sources = 3; + com.wegtam.tensei.remote.adt.ConnectionInformation target = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusMessage.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusMessage.proto new file mode 100644 index 0000000..47ec106 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusMessage.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/messages/StatusType.proto"; + +message StatusMessage { + string reporter = 1; + string message = 2; + StatusType status_type = 3; + StatusMessage cause = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusType.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusType.proto new file mode 100644 index 0000000..8e6fccd --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/StatusType.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum StatusType { + MinorError = 0; + MajorError = 1; + FatalError = 3; + NoAgentAvailable = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationAborted.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationAborted.proto new file mode 100644 index 0000000..d9065c3 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationAborted.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/messages/StatusMessage.proto"; + +message TransformationAborted { + string agent_id = 1; + string run_id = 2; + StatusMessage cause = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationCompleted.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationCompleted.proto new file mode 100644 index 0000000..144ca48 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationCompleted.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message TransformationCompleted { + string agent_id = 1; + string run_id = 2; +} \ No newline at end of file diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationFailed.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationFailed.proto new file mode 100644 index 0000000..0b692a0 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationFailed.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/messages/StatusMessage.proto"; + +message TransformationFailed { + string agent_id = 1; + string run_id = 2; + StatusMessage cause = 3; +} \ No newline at end of file diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationStarted.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationStarted.proto new file mode 100644 index 0000000..0582f4d --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/TransformationStarted.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages; + +option java_multiple_files = true; +option optimize_for = SPEED; + +message TransformationStarted { + string agent_id = 1; + string run_id = 2; +} \ No newline at end of file diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/LicenseValidationResult.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/LicenseValidationResult.proto new file mode 100644 index 0000000..ef5c7fd --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/LicenseValidationResult.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages.license; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum LicenseValidationResult { + InvalidDamaged = 0; + InvalidExpired = 1; + InvalidSignature = 2; + InvalidUnsigned = 3; + Valid = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/TenseiLicenseMessages.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/TenseiLicenseMessages.proto new file mode 100644 index 0000000..96d4dfb --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/messages/license/TenseiLicenseMessages.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.messages.license; + +import "scalapb/scalapb.proto"; +import "com/wegtam/tensei/remote/messages/license/LicenseValidationResult.proto"; + +option java_multiple_files = false; +option optimize_for = SPEED; + +option (scalapb.options) = { + // All classes that extend a sealed trait need to be in the same Scala + // file, so we set single_file to true. + single_file: true + + // Generate the base trait. + preamble: "sealed trait TenseiLicenseMessages" +}; + +message AllowedNumberOfAgents { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 count = 1; +} + +message AllowedNumberOfConfigurations { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 count = 1; +} + +message AllowedNumberOfCronjobs { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 count = 1; +} + +message AllowedNumberOfTriggers { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 count = 1; +} + +message AllowedNumberOfUsers { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 count = 1; +} + +message LicenseEntitiesData { + option (scalapb.message).extends = "TenseiLicenseMessages"; + int32 agents = 1; + int32 configurations = 2; + int32 cronjobs = 3; + int32 trigger = 4; + int32 users = 5; +} + +message LicenseExpiresIn { + option (scalapb.message).extends = "TenseiLicenseMessages"; + string period = 1; +} + +message LicenseMetaData { + option (scalapb.message).extends = "TenseiLicenseMessages"; + string id = 1; + string licensee = 2; + string period = 3; +} + +message NoLicenseInstalled { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportAllowedNumberOfAgents { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportAllowedNumberOfConfigurations { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportAllowedNumberOfCronjobs { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportAllowedNumberOfTriggers { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportAllowedNumberOfUsers { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportLicenseExpirationPeriod { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportLicenseMetaData { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message ReportLicenseEntitiesData { + option (scalapb.message).extends = "TenseiLicenseMessages"; +} + +message UpdateLicense { + option (scalapb.message).extends = "TenseiLicenseMessages"; + string encoded_license = 1; +} + +message UpdateLicenseResult { + option (scalapb.message).extends = "TenseiLicenseMessages"; + bool succeeded = 1; + string message = 2; +} + +message ValidateLicense { + option (scalapb.message).extends = "TenseiLicenseMessages"; + string encoded_license = 1; +} + +message ValidateLicenseResult { + option (scalapb.message).extends = "TenseiLicenseMessages"; + LicenseValidationResult result = 1; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentAuthorisationState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentAuthorisationState.proto new file mode 100644 index 0000000..8482739 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentAuthorisationState.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum AgentAuthorisationState { + AgentConnected = 0; + AgentDisconnected = 1; + AgentUnauthorised = 2; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentState.proto new file mode 100644 index 0000000..bacda03 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentState.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum AgentState { + AgentIdle = 0; + AgentAborting = 1; + AgentCleaningUp = 2; + AgentInitializingResources = 3; + AgentWorking = 4; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentWorkingState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentWorkingState.proto new file mode 100644 index 0000000..67e06ea --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/AgentWorkingState.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +import "com/wegtam/tensei/remote/adt/RuntimeStats.proto"; +import "com/wegtam/tensei/remote/states/AgentState.proto"; +import "com/wegtam/tensei/remote/states/ParserState.proto"; +import "com/wegtam/tensei/remote/states/ProcessorState.proto"; + +message AgentWorkingState { + string id = 1; + AgentState state = 2; + ParserState parser = 3; + ProcessorState processor = 4; + repeated com.wegtam.tensei.remote.adt.RuntimeStats runtime_stats = 5; + string unique_identifier = 6; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ParserState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ParserState.proto new file mode 100644 index 0000000..82979a6 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ParserState.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum ParserState { + ParserIdle = 0; + ParserValidatingSyntax = 1; + ParserValidatingChecksums = 2; + ParserPreparingSourceData = 3; + ParserInitializingSubParsers = 4; + ParserParsing = 5; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ProcessorState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ProcessorState.proto new file mode 100644 index 0000000..26eb46a --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ProcessorState.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum ProcessorState { + ProcessorIdle = 0; + ProcessorSorting = 1; + ProcessorProcessing = 2; + ProcessorWaitingForWriterClosing = 3; +} diff --git a/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ServerState.proto b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ServerState.proto new file mode 100644 index 0000000..b7fbfd1 --- /dev/null +++ b/messages/src/main/protobuf/com/wegtam/tensei/remote/states/ServerState.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.wegtam.tensei.remote.states; + +option java_multiple_files = true; +option optimize_for = SPEED; + +enum ServerState { + ServerBooting = 0; + ServerInitialising = 1; + ServerRunning = 2; +} \ No newline at end of file diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..c091b86 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=0.13.16 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..adaaedb --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,12 @@ +addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.1") +addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.9.3") +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "2.0.0") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.27") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.1") +addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.6") +addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.10") +addSbtPlugin("org.wartremover" % "sbt-wartremover" % "2.1.1") + +// The library dependencies are needed for the Scala Protobuf plugin: +libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.6.2" + diff --git a/src/main/scala/com/wegtam/tensei/adt/AgentAuthorizationState.scala b/src/main/scala/com/wegtam/tensei/adt/AgentAuthorizationState.scala new file mode 100644 index 0000000..7a727e7 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/AgentAuthorizationState.scala @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Defines all possible authorization states for an agent. + */ +sealed trait AgentAuthorizationState { + override def toString: String = this match { + case AgentAuthorizationState.Connected ⇒ "Connected" + case AgentAuthorizationState.Disconnected ⇒ "Disconnected" + case AgentAuthorizationState.Unauthorized ⇒ "Unauthorized" + } +} + +/** + * The concrete authorization states package into an object. + */ +object AgentAuthorizationState { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def AgentAuthorizationStateCodecJson: CodecJson[AgentAuthorizationState] = + CodecJson( + (s: AgentAuthorizationState) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "Connected" ⇒ Connected + case "Disconnected" ⇒ Disconnected + case "Unauthorized" ⇒ Unauthorized + case e: String ⇒ + throw new IllegalArgumentException(s"Unknown AgentAuthorizationState: '$e'!") + } + ) + + /** + * The agent is connected to the cluster and has a valid license (e.g. is authorized). + */ + case object Connected extends AgentAuthorizationState + + /** + * The agent is not connected to the cluster. + */ + case object Disconnected extends AgentAuthorizationState + + /** + * The agent is connected to the cluster but not authorized to work (for example it's license is not valid or has expired). + */ + case object Unauthorized extends AgentAuthorizationState + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/AgentInformation.scala b/src/main/scala/com/wegtam/tensei/adt/AgentInformation.scala new file mode 100644 index 0000000..f2df005 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/AgentInformation.scala @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Informations about an agent. + * + * @param id The unique name of the agent. + * @param path The actor path to the agent. + * @param auth The authorization state of the agent. + * @param lastUpdated The timestamp marking the last information update. + * @param workingState The working state of the agent with information of the parser status. + */ +final case class AgentInformation( + id: String, + path: String, + auth: AgentAuthorizationState, + lastUpdated: Long, + workingState: Option[AgentWorkingState] = None +) + +object AgentInformation { + + implicit def AgentInformationCodecJson: CodecJson[AgentInformation] = + CodecJson( + (i: AgentInformation) ⇒ + ("workingState" := i.workingState) ->: ("updated" := jNumber(i.lastUpdated)) ->: ("auth" := i.auth) ->: ("path" := i.path) ->: ("id" := i.id) ->: jEmptyObject, + c ⇒ + for { + id ← (c --\ "id").as[String] + path ← (c --\ "path").as[String] + auth ← (c --\ "auth").as[AgentAuthorizationState] + updated ← (c --\ "updated").as[Long] + workingState ← (c --\ "workingState").as[Option[AgentWorkingState]] + } yield + AgentInformation(id = id, + path = path, + auth = auth, + lastUpdated = updated, + workingState = workingState) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/AgentStartTransformationMessage.scala b/src/main/scala/com/wegtam/tensei/adt/AgentStartTransformationMessage.scala new file mode 100644 index 0000000..283bf7b --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/AgentStartTransformationMessage.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Message that starts a complete transformation process + * + * @param sources A list of connection informations for the source data. + * @param target Connection information for the target data + * @param cookbook The cookbook containing the dfasdls and mapping and transformation recipes for the data. + * @param uniqueIdentifier An optional Unique Identifier that is used to identify the message. + */ +final case class AgentStartTransformationMessage( + sources: List[ConnectionInformation], + target: ConnectionInformation, + cookbook: Cookbook, + uniqueIdentifier: Option[String] = None +) { + + /** + * Checks if at least one of the sources defines a checksum. + * + * @return `true` if at least one of the sources defines a checksum and `false` otherwise. + */ + def hasChecksums: Boolean = sources.exists(c ⇒ c.checksum.exists(_.length > 0)) +} + +object AgentStartTransformationMessage { + implicit def AgentStartTransformationMessageCodecJson + : CodecJson[AgentStartTransformationMessage] = + CodecJson( + (a: AgentStartTransformationMessage) ⇒ + ("uniqueIdentifier" := a.uniqueIdentifier) ->: + ("cookbook" := a.cookbook) ->: + ("sources" := a.sources) ->: + ("target" := a.target) ->: + jEmptyObject, + c ⇒ + for { + sources ← (c --\ "sources").as[List[ConnectionInformation]] + target ← (c --\ "target").as[ConnectionInformation] + cookbook ← (c --\ "cookbook").as[Cookbook] + uniqueIdentifier ← (c --\ "uniqueIdentifier").as[Option[String]] + } yield AgentStartTransformationMessage(sources, target, cookbook, uniqueIdentifier) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/AgentWorkingState.scala b/src/main/scala/com/wegtam/tensei/adt/AgentWorkingState.scala new file mode 100644 index 0000000..53cb9af --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/AgentWorkingState.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.agent.{ ParserState, ProcessorState, TenseiAgentState } + +/** + * The "working" state of an agent. + * + * @param id The ID of the agent. + * @param state The current state of the agent. + * @param parser The current state of the parser. + * @param processor The current state of the processor. + * @param runtimeStats Some statistics regarding the agent's jvm runtime. The map should use the agent's hostname as the key value. + * @param uniqueIdentifier An option to the id of the currently running transformation configuration. + */ +final case class AgentWorkingState( + id: String, + state: TenseiAgentState, + parser: ParserState, + processor: ProcessorState, + runtimeStats: Map[String, RuntimeStats], + uniqueIdentifier: Option[String] = None +) {} + +object AgentWorkingState { + implicit def AgentWorkingStateCodecJson: CodecJson[AgentWorkingState] = + CodecJson( + (s: AgentWorkingState) ⇒ + ("uniqueIdentifier" := s.uniqueIdentifier) ->: ("runtime" := s.runtimeStats) ->: ("processor" := s.processor) ->: ("parser" := s.parser) ->: ("state" := s.state) ->: ("id" := s.id) ->: jEmptyObject, + c ⇒ + for { + id ← (c --\ "id").as[String] + agentState ← (c --\ "state").as[TenseiAgentState] + parserState ← (c --\ "parser").as[ParserState] + processorState ← (c --\ "processor").as[ProcessorState] + runtime ← (c --\ "runtime").as[Map[String, RuntimeStats]] + uniqueIdentifier ← (c --\ "uniqueIdentifier").as[Option[String]] + } yield + AgentWorkingState(id = id, + state = agentState, + parser = parserState, + processor = processorState, + runtimeStats = runtime, + uniqueIdentifier = uniqueIdentifier) + ) +} diff --git a/src/main/scala/com/wegtam/tensei/adt/AtomicTransformationDescription.scala b/src/main/scala/com/wegtam/tensei/adt/AtomicTransformationDescription.scala new file mode 100644 index 0000000..4564b3d --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/AtomicTransformationDescription.scala @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * A container for an atomic transformation that is applied for a specified source element. + * + * @param element The reference to the source data element. + * @param transformerClassName The name of the class that implements the transformer. + * @param options The options for the transformer. + */ +final case class AtomicTransformationDescription( + element: ElementReference, + transformerClassName: String, + options: TransformerOptions +) + +object AtomicTransformationDescription { + + implicit def AtomicTransformationDescriptionCodecJson + : CodecJson[AtomicTransformationDescription] = + CodecJson( + (t: AtomicTransformationDescription) ⇒ + ("options" := t.options) ->: + ("transformerClassName" := t.transformerClassName.toString) ->: + ("element" := t.element) ->: + jEmptyObject, + c ⇒ + for { + element ← (c --\ "element").as[ElementReference] + transformerClassName ← (c --\ "transformerClassName").as[String] + options ← (c --\ "options").as[TransformerOptions] + } yield AtomicTransformationDescription(element, transformerClassName, options) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/ClusterConstants.scala b/src/main/scala/com/wegtam/tensei/adt/ClusterConstants.scala new file mode 100644 index 0000000..4296c51 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/ClusterConstants.scala @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +/** + * Defines serveral constants for the cluster like the system name and role names. + */ +object ClusterConstants { + val systemName = "tensei-system" + + val topLevelActorNameOnAgent = "TenseiAgent" + + val topLevelActorNameOnServer = "TenseiServer" + + object Roles { + val agent = "agent" + + val frontend = "frontend" + + val server = "server" + + val watchdog = "watchdog" + } +} diff --git a/src/main/scala/com/wegtam/tensei/adt/ConnectionInformation.scala b/src/main/scala/com/wegtam/tensei/adt/ConnectionInformation.scala new file mode 100644 index 0000000..8fddfeb --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/ConnectionInformation.scala @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import java.net.URI + +/** + * Abstract data type that holds connection information. + * + * @param uri Unique Resource Identifier describing the data location + * @param dfasdlRef An option to the `DFASDLReference` that points to the DFASDL for this connection. + * @param username Username (optional) + * @param password Password (optional) + * @param checksum An optional SHA256 checksum of the input data + */ +final case class ConnectionInformation( + uri: URI, + dfasdlRef: Option[DFASDLReference], + username: Option[String] = None, + password: Option[String] = None, + checksum: Option[String] = None, + languageTag: Option[String] = None +) + +object ConnectionInformation { + + implicit def ConnectionInformationCodecJson: CodecJson[ConnectionInformation] = + CodecJson( + (c: ConnectionInformation) ⇒ + ("languageTag" := c.languageTag) ->: + ("checksum" := c.checksum) ->: + ("password" := c.password) ->: + ("username" := c.username) ->: + ("dfasdlRef" := c.dfasdlRef) ->: + ("uri" := c.uri.toString) ->: + jEmptyObject, + c ⇒ + for { + uri ← (c --\ "uri").as[String] + dfasdlRef ← (c --\ "dfasdlRef").as[Option[DFASDLReference]] + username ← (c --\ "username").as[Option[String]] + password ← (c --\ "password").as[Option[String]] + checksum ← (c --\ "checksum").as[Option[String]] + languageTag ← (c --\ "languageTag").as[Option[String]] + } yield + ConnectionInformation(new URI(uri), dfasdlRef, username, password, checksum, languageTag) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/Cookbook.scala b/src/main/scala/com/wegtam/tensei/adt/Cookbook.scala new file mode 100644 index 0000000..baa2c1f --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/Cookbook.scala @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * A cookbook contains a list of source dfasdl descriptions, a target dfasdl description + * and a list of recipes describing the actual mapping of the sources into the target. + * + * @param id The cookbook identifier (ID). + * @param sources A list of DFASDL descriptions. + * @param target A DFASDL description. + * @param recipes A list of recipes (see `Recipe`). + */ +@SuppressWarnings(Array("org.wartremover.warts.Null")) +final case class Cookbook( + id: String, + sources: List[DFASDL], + target: Option[DFASDL], + recipes: List[Recipe] +) { + require(id != null, "The cookbook ID must not be null!") + require(id.length > 0, "The cookbook ID must not be empty!") + + /** + * Returns an option to the DFASDL that is referenced by the given `DFASDLReference`. + * If the cookbook id doesn't match then `None` is returned. + * + * @param ref A reference to a DFASDL. + * @return An option to the DFASDL. + */ + def findDFASDL(ref: DFASDLReference): Option[DFASDL] = + if (ref.cookbookId == id) { + // We try the cheap match first. + if (target.exists(_.id == ref.dfasdlId)) + target + else + sources.find(_.id == ref.dfasdlId) // Search through source dfasdls. + } else + None // Wrong cookbook. + + /** + * A set of source elements that are used within mappings. + */ + lazy val usedSourceIds: Set[ElementReference] = + recipes.flatMap(recipe ⇒ recipe.mappings.flatMap(mapping ⇒ mapping.sources)).toSet +} + +object Cookbook { + + implicit def CookbookCodecJson: CodecJson[Cookbook] = + CodecJson( + (c: Cookbook) ⇒ + ("recipes" := c.recipes) ->: ("target" := c.target) ->: ("sources" := c.sources) ->: ("id" := c.id) ->: jEmptyObject, + cursor ⇒ + for { + id ← (cursor --\ "id").as[String] + sources ← (cursor --\ "sources").as[List[DFASDL]] + target ← (cursor --\ "target").as[Option[DFASDL]] + recipes ← (cursor --\ "recipes").as[List[Recipe]] + } yield Cookbook(id, sources, target, recipes) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/DFASDL.scala b/src/main/scala/com/wegtam/tensei/adt/DFASDL.scala new file mode 100644 index 0000000..697982c --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/DFASDL.scala @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import java.io.{ StringReader, StringWriter, Writer } +import javax.xml.parsers.DocumentBuilderFactory +import javax.xml.transform.dom.DOMSource +import javax.xml.transform.stream.StreamResult +import javax.xml.transform.{ OutputKeys, TransformerFactory } + +import argonaut._, Argonaut._ +import org.w3c.dom.Element +import org.w3c.dom.traversal.{ DocumentTraversal, NodeFilter } +import org.xml.sax.InputSource + +import scala.collection.mutable.ListBuffer + +/** + * A wrapper class for DFASDL descriptions. + * + * @param id The identifier (e.g. ID) for the DFASDL. + * @param content A string holding the DFASDL. + * @param version The version of the DFASDL, defaults to `1.0-SNAPSHOT` + */ +final case class DFASDL( + id: String, + content: String, + version: String = "1.0-SNAPSHOT" +) { + + override def equals(obj: scala.Any): Boolean = + obj match { + case other: DFASDL ⇒ + id == other.id && content == other.content && version == other.version + case _ ⇒ false + } + + override def hashCode(): Int = + 431 + 7 * id.hashCode + 7 * content.hashCode + 7 * version.hashCode + +} + +object DFASDL { + + // The prefix used for auto-generated IDs. + val AUTO_ID_PREFIX = "auto-id" + val AUTO_ID_REGEX = s"$AUTO_ID_PREFIX-(\\d+)".r + val DFASDL_ROOT_ELEMENT = "dfasdl" // FIXME We should use the dfasdl helper library here (if it is ready)! + + implicit def DFASDLCodecJson: CodecJson[DFASDL] = + CodecJson( + (dfasdl: DFASDL) ⇒ + ("id" := dfasdl.id) ->: ("content" := dfasdl.content) ->: ("version" := dfasdl.version) ->: jEmptyObject, + cursor ⇒ + for { + id ← (cursor --\ "id").as[String] + content ← (cursor --\ "content").as[String] + version ← (cursor --\ "version").as[String] + } yield DFASDL(id, content, version) + ) + + /** + * Takes a dfasdl and fills missing ID fields with auto-generated values. + * + * @param dfasdl A DFASDL. + * @param indentXml A flag that indicates if the returned xml should be formatted human readable. + * @return A DFASDL with no empty ID fields. + */ + @SuppressWarnings(Array("org.wartremover.warts.Null")) + def autogenerateMissingIds(dfasdl: DFASDL, indentXml: Boolean = false): DFASDL = + if (dfasdl.content.isEmpty) + dfasdl + else { + // Parse xml string into dom and do our work. + val documentBuilderFactory = DocumentBuilderFactory.newInstance() + val documentBuilder = documentBuilderFactory.newDocumentBuilder() + val xmlTree = documentBuilder.parse(new InputSource(new StringReader(dfasdl.content))) + xmlTree.getDocumentElement.normalize() + val elementList: ListBuffer[Element] = new ListBuffer[Element]() + val traversal = xmlTree.asInstanceOf[DocumentTraversal] + val iterator = traversal.createNodeIterator(xmlTree.getDocumentElement, + NodeFilter.SHOW_ELEMENT, + null, + true) + var needsAutoIds = false + var currentNode = iterator.nextNode() + var idCounter = 0L + while (currentNode != null) { + val e = currentNode.asInstanceOf[Element] + elementList += e // Buffer element + // Try to examine the largest existing auto-id counter. + if (e.hasAttribute("id")) + e.getAttribute("id") match { + case AUTO_ID_REGEX(counter) ⇒ + // An auto-generated ID. + val cnt = counter.toLong + if (cnt > idCounter) idCounter = cnt + case _ ⇒ + // A regular ID. + } + // If we aren't the root element we must have an id. + if (e.getTagName != DFASDL_ROOT_ELEMENT) + needsAutoIds = + if (!e.hasAttribute("id") || e.getAttribute("id").isEmpty) + true + else + false + currentNode = iterator.nextNode() + } + // Only loop again if neccessary. + if (needsAutoIds) { + elementList.foreach( + e ⇒ + if (e.getTagName != DFASDL_ROOT_ELEMENT && (!e + .hasAttribute("id") || e.getAttribute("id").isEmpty)) { + idCounter = idCounter + 1 + e.setAttribute("id", s"$AUTO_ID_PREFIX-$idCounter") + } + ) + val transformer = TransformerFactory.newInstance().newTransformer() + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8") + if (indentXml) transformer.setOutputProperty(OutputKeys.INDENT, "yes") + val out: Writer = new StringWriter() + transformer.transform(new DOMSource(xmlTree), new StreamResult(out)) + dfasdl.copy(content = out.toString) + } else + dfasdl + } + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/DFASDLReference.scala b/src/main/scala/com/wegtam/tensei/adt/DFASDLReference.scala new file mode 100644 index 0000000..236c22f --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/DFASDLReference.scala @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * A reference for a specific dfasdl within a specific cookbook. + * + * @param cookbookId The ID of the cookbook that contains the DFASDL. + * @param dfasdlId The ID of the DFASDL. + */ +@SuppressWarnings(Array("org.wartremover.warts.Null")) +final case class DFASDLReference(cookbookId: String, dfasdlId: String) { + + require(cookbookId != null, "The cookbook ID must not be null!") + require(cookbookId.length > 0, "The cookbook ID must not be empty!") + require(dfasdlId != null, "The DFASDL ID must not be null!") + require(dfasdlId.length > 0, "The DFASDL ID must not be empty!") + +} + +object DFASDLReference { + + implicit def DFASDLReferenceCodecJson: CodecJson[DFASDLReference] = + CodecJson( + (r: DFASDLReference) ⇒ + ("dfasdl-id" := r.dfasdlId) ->: ("cookbook-id" := r.cookbookId) ->: jEmptyObject, + cursor ⇒ + for { + cid ← (cursor --\ "cookbook-id").as[String] + did ← (cursor --\ "dfasdl-id").as[String] + } yield DFASDLReference(cid, did) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/ElementReference.scala b/src/main/scala/com/wegtam/tensei/adt/ElementReference.scala new file mode 100644 index 0000000..49966c1 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/ElementReference.scala @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * A reference to a specific DFASDL element. + * This is needed to reference elements with the same id accross multiple + * DFASDL documents. + * + * @param dfasdlId The ID of the DFASDL that includes the element. + * @param elementId The ID of the element within the DFASDL. + */ +@SuppressWarnings(Array("org.wartremover.warts.Null")) +final case class ElementReference(dfasdlId: String, elementId: String) { + + require(dfasdlId != null, "The DFASDL ID must not be null!") + require(dfasdlId.length > 0, "The DFASDL ID must not be empty!") + require(elementId != null, "The element ID must not be null!") + require(elementId.length > 0, "The element ID must not be empty!") + +} + +object ElementReference { + + implicit def ElementReferenceCodecJson: CodecJson[ElementReference] = + CodecJson( + (r: ElementReference) ⇒ + ("dfasdlId" := r.dfasdlId) ->: ("elementId" := r.elementId) ->: jEmptyObject, + c ⇒ + for { + did ← (c --\ "dfasdlId").as[String] + eid ← (c --\ "elementId").as[String] + } yield ElementReference(dfasdlId = did, elementId = eid) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/ExtractSchemaOptions.scala b/src/main/scala/com/wegtam/tensei/adt/ExtractSchemaOptions.scala new file mode 100644 index 0000000..c6bd748 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/ExtractSchemaOptions.scala @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +/** + * Options that can be passed to the schema extractor. + * + * @param csvHeader A flag that indicates if a possible CSV file has a header line. + * @param csvSeparator The separator character used in a CSV file. + * @param encoding The encoding of the file specified as the [[java.nio.charset.Charset]] name. + * @todo This generic options class will be a dumping ground for all kinds of options. Research a way to clean/split this up more properly. + */ +final case class ExtractSchemaOptions( + csvHeader: Boolean, + csvSeparator: Option[String], + encoding: Option[String] +) + +object ExtractSchemaOptions { + + /** + * Create options for a schema extraction from a CSV file. + * + * @param hasHeaderLine Set this to `true` if the first line of the file contains the column headers. + * @param separator The character that is used to separate columns (`,;\t`). + * @param encoding The encoding of the file specified as the [[java.nio.charset.Charset]] name. + * @return Options for the schema extractor. + */ + def createCsvOptions(hasHeaderLine: Boolean, + separator: String, + encoding: String): ExtractSchemaOptions = ExtractSchemaOptions( + csvHeader = hasHeaderLine, + csvSeparator = if (separator.isEmpty) None else Option(separator), + encoding = if (encoding.isEmpty) None else Option(encoding) + ) + + /** + * Create options for a schema extraction from a database. + * + * @return Options for the schema extractor. + */ + def createDatabaseOptions(): ExtractSchemaOptions = ExtractSchemaOptions( + csvHeader = false, + csvSeparator = None, + encoding = None + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/GlobalMessages.scala b/src/main/scala/com/wegtam/tensei/adt/GlobalMessages.scala new file mode 100644 index 0000000..71519f6 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/GlobalMessages.scala @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import akka.actor.ActorRef + +import scalaz._ + +/** + * Global messages that can be send accross the cluster. + */ +sealed trait GlobalMessages + +/** + * Container object for the global messages to keep the namespace somewhat clean. + */ +object GlobalMessages { + + final case class AbortTransformation(ref: ActorRef) extends GlobalMessages + + final case class AbortTransformationResponse(ref: ActorRef, message: Option[String]) + extends GlobalMessages + + /** + * This message can be send to notify someone that an error has occured. + * + * @param error The actual error. + */ + final case class ErrorOccured(error: StatusMessage) extends GlobalMessages + + /** + * Contains a bulk of error messages. + * + * @param errors A list of error messages. + */ + final case class ErrorsOccured(errors: List[ErrorOccured]) extends GlobalMessages + + /** + * This message can be send to notify someone that an exception has occured. + * + * @param ref An option to the actor closest to the exception. This is usually the one that catched the exception. + * @param exception The exception. + */ + final case class ExceptionOccured(ref: Option[ActorRef], exception: Throwable) + extends GlobalMessages + + /** + * Try to extract the schema from the data source with the given connection information + * into a dfasdl. + * + * @param source The connection information for the data source. + * @param options The options for the schema extractor. + */ + final case class ExtractSchema( + source: ConnectionInformation, + options: ExtractSchemaOptions + ) extends GlobalMessages + + /** + * The result of a schema extraction request. + * + * @param source The connection information that was used for the data source. + * @param result The actual result which maybe either a `DFASDL` or a string holding an error message. + */ + final case class ExtractSchemaResult(source: ConnectionInformation, result: String \/ DFASDL) + extends GlobalMessages + + final case class ReportingTo(ref: ActorRef, id: Option[String] = None) extends GlobalMessages + + case object ReportToCaller extends GlobalMessages + + final case class ReportToRef(ref: ActorRef) extends GlobalMessages + + /** + * Request the log files of the specified transformation run from an agent. + * + * @param agentId The id of the agent that should be queried. + * @param uuid The agent run identifier which is usually uuid. + * @param offset An optional offset given in number of bytes that should be skipped. + * @param maxSize An optional number of bytes that specifies the maximum size of logs that should be sent. + */ + final case class RequestAgentRunLogs( + agentId: String, + uuid: String, + offset: Option[Long] = None, + maxSize: Option[Long] = None + ) extends GlobalMessages + + /** + * Request the meta data of the specified transformation run from an agent. + * + * @param uuid The agent run identifier which is usually uuid. + */ + final case class RequestAgentRunLogsMetaData(uuid: String) extends GlobalMessages + + /** + * Report the meta data for requested run logs. + * + * @param agentId The id of the agent that holds the logs. + * @param uuid The agent run identifier which is usually uuid. + * @param size The size of the log file(s) in bytes. + */ + final case class ReportAgentRunLogsMetaData( + agentId: String, + uuid: String, + size: Long + ) extends GlobalMessages + + /** + * Report a line from the run logs. + * + * @param uuid The agent run identifier which is usually uuid. + * @param logLine The actual log entry. + * @param offet The offset within the log file that marks the start of the log entry. + */ + final case class ReportAgentRunLogLine(uuid: String, logLine: String, offet: Long) + extends GlobalMessages + + case object Restart extends GlobalMessages + + case object Shutdown extends GlobalMessages + + final case class TransformationAborted(uuid: Option[String] = None) extends GlobalMessages + + final case class TransformationCompleted(uuid: Option[String] = None) extends GlobalMessages + + final case class TransformationError(uuid: Option[String] = None, error: StatusMessage) + extends GlobalMessages + + final case class TransformationStarted(uuid: Option[String] = None) extends GlobalMessages + + final case class UnwatchActor(ref: ActorRef) extends GlobalMessages + + final case class WatchActor(ref: ActorRef) extends GlobalMessages + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/LicenseValidationResult.scala b/src/main/scala/com/wegtam/tensei/adt/LicenseValidationResult.scala new file mode 100644 index 0000000..dd2497d --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/LicenseValidationResult.scala @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +/** + * All possible results of a license validation. + */ +sealed trait LicenseValidationResult + +/** + * A companion object to keep the namespace clean. + */ +object LicenseValidationResult { + + /** + * The license is not valid. + * + * @param reason An option to a reason why the license is invalid. + */ + final case class Invalid(reason: Option[InvalidLicenseReason]) extends LicenseValidationResult + + /** + * The license is valid. + */ + case object Valid extends LicenseValidationResult + +} + +/** + * All possible reasons for an invalid license. + */ +sealed trait InvalidLicenseReason + +/** + * A companion object to hold the invalid license reasons to keep the namespace clean. + */ +object InvalidLicenseReason { + + /** + * This indicates that the license data was damaged e.g. not complete and + * we were unable to process it. + */ + case object Damaged extends InvalidLicenseReason + + /** + * The license has expired. + */ + case object Expired extends InvalidLicenseReason + + /** + * The signature on the license is invalid. + */ + case object InvalidSignature extends InvalidLicenseReason + + /** + * The license was missing a signature. + */ + case object Unsigned extends InvalidLicenseReason + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/MappingKeyFieldDefinition.scala b/src/main/scala/com/wegtam/tensei/adt/MappingKeyFieldDefinition.scala new file mode 100644 index 0000000..7688b2d --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/MappingKeyFieldDefinition.scala @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Defines a "key field" for an element id. This is necessary if we are mapping elements from + * multiple data sources together. + * The source element values are extracted from the sequence rows where the key field values + * are equal. + * A key field has to have the same name in all relevant data sources. + * + * @param name The name of the field that is used as a key field. + */ +final case class MappingKeyFieldDefinition(name: String) + +object MappingKeyFieldDefinition { + + implicit def MappingKeyFieldDefinitionCodecJson: CodecJson[MappingKeyFieldDefinition] = + CodecJson( + (m: MappingKeyFieldDefinition) ⇒ ("name" := m.name) ->: jEmptyObject, + cursor ⇒ + for { + name ← (cursor --\ "name").as[String] + } yield MappingKeyFieldDefinition(name) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/MappingTransformation.scala b/src/main/scala/com/wegtam/tensei/adt/MappingTransformation.scala new file mode 100644 index 0000000..63a0d1a --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/MappingTransformation.scala @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * A mapping transformation describes the mapping of the data hold in elements specified by one or more + * source elements to elements in the target tree. + * + * The simplest possible case is the mapping of one source elemenht to one target element without any transformations. + * + * If more than one element is specified in MapAllToAll mode: + * All given sources are always mapped to all given targets using + * the specified transformations. This means that you have to define aggregation transformations if you use + * multiple sources (for example a concatenation transformation for multiple string sources). + * + * @param sources A list of source elements. + * @param targets A list of target elements. + * @param transformations A list of transformations descriptions. + * @param atomicTransformations A list of atomic transformation descriptions. + * @param mappingKey The key field that is used for mapping from different data sources. + */ +final case class MappingTransformation( + sources: List[ElementReference], + targets: List[ElementReference], + transformations: List[TransformationDescription] = List(), + atomicTransformations: List[AtomicTransformationDescription] = List(), + mappingKey: Option[MappingKeyFieldDefinition] = None +) { + + require(targets.nonEmpty, "You have to define at least 1 target id!") + +} + +object MappingTransformation { + + implicit def MappingTransformationCodecJson: CodecJson[MappingTransformation] = + CodecJson( + (m: MappingTransformation) ⇒ + ("mappingKey" := m.mappingKey) ->: + ("atomicTransformations" := m.atomicTransformations) ->: + ("transformations" := m.transformations) ->: + ("targets" := m.targets) ->: + ("sources" := m.sources) ->: + jEmptyObject, + c ⇒ + for { + sources ← (c --\ "sources").as[List[ElementReference]] + targets ← (c --\ "targets").as[List[ElementReference]] + transformations ← (c --\ "transformations").as[List[TransformationDescription]] + atomicTransformations ← (c --\ "atomicTransformations") + .as[List[AtomicTransformationDescription]] + mappingKeys ← (c --\ "mappingKey").as[Option[MappingKeyFieldDefinition]] + } yield + MappingTransformation(sources, + targets, + transformations, + atomicTransformations, + mappingKeys) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/Recipe.scala b/src/main/scala/com/wegtam/tensei/adt/Recipe.scala new file mode 100644 index 0000000..fb53f5c --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/Recipe.scala @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.adt.Recipe.{ MapAllToAll, RecipeMode } + +/** + * A recipe describes the mapping of the source data tree into the target data tree by + * mapping transformations. + * + * @param id An unique id to identify the recipe. + * @param mode Define the mapping mode for the recipe. + * @param mappings A list of mapping transformations. + */ +@SuppressWarnings(Array("org.wartremover.warts.Null")) +final case class Recipe( + id: String, + mode: RecipeMode = MapAllToAll, + mappings: List[MappingTransformation] +) { + + require(id != null, "The ID of a recipe must not be null!") + require(id.length > 0, "The ID of a recipe must not be empty!") + +} + +object Recipe { + + implicit def RecipeCodecJson: CodecJson[Recipe] = + CodecJson( + (r: Recipe) ⇒ + ("mappings" := r.mappings) ->: + ("mode" := r.mode) ->: + ("id" := r.id) ->: + jEmptyObject, + c ⇒ + for { + id ← (c --\ "id").as[String] + mode ← (c --\ "mode").as[RecipeMode] + mappings ← (c --\ "mappings").as[List[MappingTransformation]] + } yield Recipe(id, mode, mappings) + ) + + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def RecipeModeCodecJson: CodecJson[RecipeMode] = + CodecJson( + (m: RecipeMode) ⇒ + m match { + case MapAllToAll ⇒ jString("MapAllToAll") + case MapOneToOne ⇒ jString("MapOneToOne") + }, + cursor ⇒ + for { + modeName ← cursor.as[String] + } yield { + modeName match { + case "MapAllToAll" ⇒ MapAllToAll + case "MapOneToOne" ⇒ MapOneToOne + case _ ⇒ throw new IllegalArgumentException(s"Unknown recipe mode name $modeName!") + } + } + ) + + sealed trait RecipeMode + + case object MapAllToAll extends RecipeMode + + case object MapOneToOne extends RecipeMode + + /** + * Helper function to create a recipe using the `AllToAll` mapping mode. + * + * @param id An unique id to identify the recipe. + * @param mappings A list of mapping transformations. + * @return A recipe. + */ + def createAllToAllRecipe(id: String, mappings: List[MappingTransformation]): Recipe = + new Recipe(id, MapAllToAll, mappings) + + def createOneToOneRecipe(id: String, mappings: List[MappingTransformation]): Recipe = + new Recipe(id, MapOneToOne, mappings) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/RuntimeStats.scala b/src/main/scala/com/wegtam/tensei/adt/RuntimeStats.scala new file mode 100644 index 0000000..f1daaa2 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/RuntimeStats.scala @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Some statistics regarding the jvm runtime. + * + * @param freeMemory The amount of free memory in bytes. + * @param maxMemory The maximum possible amount of memory available to the jvm in bytes. + * @param totalMemory The total available memory for the jvm in bytes. + */ +final case class RuntimeStats( + freeMemory: Long, + maxMemory: Long, + totalMemory: Long, + processors: Int = 1, + systemLoad: Option[Double] = None +) + +object RuntimeStats { + + implicit def RuntimeStatsCodecJson: CodecJson[RuntimeStats] = + CodecJson( + (s: RuntimeStats) ⇒ + ("load" := s.systemLoad) ->: ("processors" := jNumber(s.processors)) ->: ("total" := jNumber( + s.totalMemory + )) ->: ("max" := jNumber(s.maxMemory)) ->: ("free" := jNumber(s.freeMemory)) ->: jEmptyObject, + c ⇒ + for { + free ← (c --\ "free").as[Long] + max ← (c --\ "max").as[Long] + total ← (c --\ "total").as[Long] + p ← (c --\ "processors").as[Int] + load ← (c --\ "load").as[Option[Double]] + } yield + RuntimeStats(freeMemory = free, + maxMemory = max, + totalMemory = total, + processors = p, + systemLoad = load) + ) + + /** + * Creates a runtime stats object holding the stats of the current runtime. + * + * @return The current runtime stats. + */ + def getCurrentRuntimeStats: RuntimeStats = { + val runtime = Runtime.getRuntime + val osBean = java.lang.management.ManagementFactory.getOperatingSystemMXBean + new RuntimeStats( + freeMemory = runtime.freeMemory(), + maxMemory = runtime.maxMemory(), + totalMemory = runtime.totalMemory(), + processors = osBean.getAvailableProcessors, + systemLoad = Option(osBean.getSystemLoadAverage) + ) + } + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/StatsMessages.scala b/src/main/scala/com/wegtam/tensei/adt/StatsMessages.scala new file mode 100644 index 0000000..7194b62 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/StatsMessages.scala @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import scalaz._ + +sealed trait StatsMessages + +/** + * Container object for the stats messages to keep the namespace clean. + */ +object StatsMessages { + + /** + * A start message for the calculation of statistics regarding the delivered source. + * + * @param source The source specifies the DFASDL that must be used from the cookbook for the analysis. + * @param cookbook The cookbook that holds the DFASDL for the analysis. + * @param sourceIds A list of IDs that are relevant for the analysis. + * @param percent Defines the amount of that data that should be used from the source for the analysis. + */ + final case class CalculateStatistics( + source: ConnectionInformation, + cookbook: Cookbook, + sourceIds: List[String], + percent: Int = 100 + ) extends StatsMessages + + /** + * The results of the calculation regarding the delivered source. + * + * @param results A list of results of the statistical calculation for the single fields. + * @param source The source specifies the DFASDL that must be used from the cookbook for the analysis. + * @param cookbook The cookbook that holds the DFASDL for the analysis. + * @param sourceIds A list of IDs that are relevant for the analysis. + * @param percent Defines the amount of that data that should be used from the source for the analysis. + */ + final case class CalculateStatisticsResult( + results: String \/ List[StatsResult], + source: ConnectionInformation, + cookbook: Cookbook, + sourceIds: List[String], + percent: Int = 100 + ) extends StatsMessages +} + +sealed trait StatsResult + +object StatsResult { + + /** + * The basic statistical result of a statistical analyzer after the analysis of all rows. + * Depending on the Analyzer, these basic statistics mean: + * - NumericAnalyzer: Basic statistics of the numerical value. + * - StringAnalyzer : Basic statistics of the length of the strings. + * + * @param total The total amount of data that was analyzed by the analyzer including eventual errors. + * @param quantity The total amount of data that were considered in the statistical analysis without errors. + * @param min The minimum value that occured in the analyzed data. + * @param max The maximum value that occured in the analyzed data. + * @param mean The mean value of the analyzed data. + * @param errors Errors that occured during the analysis of the data. + */ + final case class BasicStatisticsResult( + total: Long = 0, + quantity: Option[Long] = None, + min: Option[Double] = None, + max: Option[Double] = None, + mean: Option[Double] = None, + errors: Option[StatisticErrors] = None + ) extends StatsResult + + /** + * A class for the different errors that can occur during the basic statistical analysis of the data. + * + * @param formatErrors Number of format errors during the casting of the data. + * @param nullErrors Number of null pointer exception errors during the casting of the data. + * @param unexpectedErrors Number of unexpected errors that are not categorisable. + */ + final case class StatisticErrors( + formatErrors: Long = 0, + nullErrors: Long = 0, + unexpectedErrors: Long = 0 + ) extends StatsResult + + /** + * A statistical result of a numerical field. + * + * @param elementId The ID of the element in the DFASDL. + * @param basic The basic statistical calculations for a numerical field. + */ + final case class StatsResultNumeric(elementId: String, basic: BasicStatisticsResult) + extends StatsResult + + /** + * A statistical result of a string field. + * + * @param elementId The ID of the element in the DFASDL. + * @param basic The basic statistical calculations for a string field. + */ + final case class StatsResultString(elementId: String, basic: BasicStatisticsResult) + extends StatsResult + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/StatusMessage.scala b/src/main/scala/com/wegtam/tensei/adt/StatusMessage.scala new file mode 100644 index 0000000..da74867 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/StatusMessage.scala @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import akka.actor.ActorRef + +/** + * A base for status messages. + * + * @param reporter An option to the reporter of the message, usually the actor path to the actor that produced the error. + * @param message The actual status message which should be a bit verbose. + * @param statusType The type of the status + * @param cause An option to another status. + */ +case class StatusMessage( + reporter: Option[String], + message: String, + statusType: StatusType = StatusType.MinorError, + cause: Option[StatusMessage] +) { + + override def toString: String = + s"""$statusType Status: $message ${cause.fold("")(c ⇒ s"""(CAUSED BY $c) """)}(REPORTED BY ${reporter + .getOrElse("None")})""" + + override def equals(obj: scala.Any): Boolean = + obj match { + case other: StatusMessage ⇒ + reporter == other.reporter && message == other.message && statusType == other.statusType && cause == other.cause + case _ ⇒ false + } + + override def hashCode(): Int = + 11 + 5 * reporter.hashCode + 5 * message.hashCode + 5 * statusType.hashCode() + 5 * cause + .hashCode() + +} + +object StatusMessage { + + /** + * Encode the status message into a json string. + * + * @return A json representation of the status message. + */ + implicit def ErrorMessageEncodeJson: EncodeJson[StatusMessage] = + EncodeJson( + (m: StatusMessage) ⇒ + ("cause" := m.cause) ->: ("statusType" := m.statusType) ->: ("message" := jString( + m.message + )) ->: ("reporter" := m.reporter) ->: jEmptyObject + ) + + /** + * Decodes generic status messages from a given json source. + * + * @return The deocded status message. + */ + implicit def ErrorMessageDecodeJson: DecodeJson[StatusMessage] = + DecodeJson( + cursor ⇒ + for { + reporter ← (cursor --\ "reporter").as[Option[String]] + message ← (cursor --\ "message").as[String] + severity ← (cursor --\ "statusType").as[StatusType] + cause ← (cursor --\ "cause").as[Option[StatusMessage]] + } yield new StatusMessage(reporter, message, severity, cause) + ) + + /** + * A helper function that returns the actor path (including the actor's address) as an option of string. + * + * @param ref An option to an actor ref. + * @return The option containing the path as string or `None` if no actor ref was given. + */ + def getReporterStringFromActorRef(ref: Option[ActorRef]): Option[String] = + ref.map(r ⇒ r.path.toSerializationFormatWithAddress(r.path.address)) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/StatusType.scala b/src/main/scala/com/wegtam/tensei/adt/StatusType.scala new file mode 100644 index 0000000..163c6dc --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/StatusType.scala @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * The type of a status message. + */ +sealed trait StatusType { + + override def toString: String = this match { + case StatusType.MinorError ⇒ "MinorError" + case StatusType.MajorError ⇒ "MajorError" + case StatusType.NoAgentAvailable ⇒ "NoAgentAvailable" + case StatusType.FatalError ⇒ "FatalError" + } + +} + +object StatusType { + + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def StatusTypeCodecJson: CodecJson[StatusType] = + CodecJson( + (s: StatusType) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "MinorError" ⇒ MinorError + case "MajorError" ⇒ MajorError + case "NoAgentAvailable" ⇒ NoAgentAvailable + case "FatalError" ⇒ FatalError + case e: String ⇒ throw new IllegalArgumentException(s"Unknown StatusType: '$e'!") + } + ) + + case object MinorError extends StatusType + + case object MajorError extends StatusType + + case object NoAgentAvailable extends StatusType + + case object FatalError extends StatusType + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/TenseiLicense.scala b/src/main/scala/com/wegtam/tensei/adt/TenseiLicense.scala new file mode 100644 index 0000000..dea2ef4 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/TenseiLicense.scala @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import java.time.{ LocalDate, Period } + +/** + * This class describes the content of a tensei license file. + * + * @param id The unique id of the license. + * @param licensee The name of the licensee (usually a company name). + * @param agents The number of agents that the licensee is allowed to use. + * @param users The number of users that the licensee is allowed to create and use in the frontend. A value of `Int.MaxValue` indicates unlimited. + * @param configurations The number of transformation configurations that the licensee is allowed to create and use in the frontend. A value of `Int.MaxValue` indicates unlimited. + * @param cronjobs The number of cronjobs that the licensee is allowed to create and use in the frontend. A value of `Int.MaxValue` indicates unlimited. + * @param triggers The number of triggers that the licensee is allowed to create and use in the frontend. A value of `Int.MaxValue` indicates unlimited. + * @param expirationDate The expiration date of the license. + */ +final case class TenseiLicense( + id: String, + licensee: String, + agents: Int, + users: Int, + configurations: Int, + cronjobs: Int, + triggers: Int, + expirationDate: LocalDate +) { + + /** + * Returns a `java.time.Period` that indicates when this license will expire. + * + * @return The period until this license will expire. + */ + def expiresIn: Period = + Period.between(LocalDate.now(), expirationDate) + +} + +/** + * The companion object holds the implicit json codecs for argonaut and several helper functions. + */ +object TenseiLicense { + + /** + * A helper codec for decoding and encoding a `java.time.LocalDate`. + * + * @return The implicit definition for argonaut how to de- and encode a local date. + */ + implicit def LocalDateCodecJson: CodecJson[LocalDate] = + CodecJson( + (d: LocalDate) ⇒ jString(d.toString), + cursor ⇒ + for { + dateString ← cursor.as[String] + } yield LocalDate.parse(dateString) + ) + + /** + * The argonaut codec for decoding and encoding a tensei license. + * + * @return The implicit definition for argonaut. + */ + implicit def TenseiLicenseCodecJson: CodecJson[TenseiLicense] = + CodecJson( + (l: TenseiLicense) ⇒ + ("playload" := generatePayload()) + ->: ("expiration-date" := l.expirationDate) + ->: ("triggers" := l.triggers) + ->: ("cronjobs" := l.cronjobs) + ->: ("configurations" := l.configurations) + ->: ("users" := l.users) + ->: ("agents" := l.agents) + ->: ("licensee" := l.licensee) + ->: ("id" := l.id) + ->: jEmptyObject, + cursor ⇒ + for { + id ← (cursor --\ "id").as[String] + licensee ← (cursor --\ "licensee").as[String] + agents ← (cursor --\ "agents").as[Int] + users ← (cursor --\ "users").as[Int] + configurations ← (cursor --\ "configurations").as[Int] + cronjobs ← (cursor --\ "cronjobs").as[Int] + triggers ← (cursor --\ "triggers").as[Int] + expirationDate ← (cursor --\ "expiration-date").as[LocalDate] + } yield + TenseiLicense(id = id, + licensee = licensee, + agents = agents, + users = users, + configurations = configurations, + cronjobs = cronjobs, + triggers = triggers, + expirationDate = expirationDate) + ) + + /** + * Create a professional license for the given licensee and expiration date. + * It throws an `IllegalArgumentException` if the given expiration date lies in the past. + * + * @param licensee The name of the licensee (usually a company name). + * @param expirationDate The expiration date of the license. + * @return A tensei professional license. + */ + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + def createProfessionalLicense(licensee: String, expirationDate: LocalDate): TenseiLicense = { + if (expirationDate.compareTo(LocalDate.now()) < 0) + throw new IllegalArgumentException("Expiration date lies in the past!") + + TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = expirationDate + ) + } + + /** + * Create an enterprise license for the given licensee and expiration date. + * It throws an `IllegalArgumentException` if the given expiration date lies in the past. + * + * @param licensee The name of the licensee (usually a company name). + * @param expirationDate The expiration date of the license. + * @return A tensei enterprise license. + */ + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + def createEnterpriseLicense(licensee: String, expirationDate: LocalDate): TenseiLicense = { + if (expirationDate.compareTo(LocalDate.now()) < 0) + throw new IllegalArgumentException("Expiration date lies in the past!") + + TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 5, + users = Int.MaxValue, + configurations = Int.MaxValue, + cronjobs = Int.MaxValue, + triggers = Int.MaxValue, + expirationDate = expirationDate + ) + } + + /** + * Create a test (or demo) license which is currently equal to a professional + * license except that is expires within a month. + * + * @param licensee The name of the licensee (usually a company name). + * @return A tensei license for test or demo systems that expires within a month. + */ + def createTestLicense(licensee: String): TenseiLicense = + createProfessionalLicense(licensee, LocalDate.now().plusMonths(1L)) + + /** + * Generates the playload field for the license using a predefined payload length. + * + * @return A random string with the desired length. + */ + def generatePayload(): String = generatePayload(4096) + + /** + * Generates the playload field for the license. + * + * @param length The desired length of the payload field. + * @return A random string with the desired length. + */ + def generatePayload(length: Int): String = scala.util.Random.alphanumeric.take(length).mkString + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/TenseiLicenseMessages.scala b/src/main/scala/com/wegtam/tensei/adt/TenseiLicenseMessages.scala new file mode 100644 index 0000000..ae951fc --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/TenseiLicenseMessages.scala @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import java.time.Period + +import scalaz._ + +/** + * Messages related to the handling of tensei licenses. + */ +sealed trait TenseiLicenseMessages + +/** + * Container object for the license messages to keep the namespace clean. + */ +object TenseiLicenseMessages { + + /** + * Reports the allowed number of agents. + * + * @param count The allowed number of agents. + */ + final case class AllowedNumberOfAgents(count: Int) extends TenseiLicenseMessages + + /** + * Reports the allowed number of transformation configurations. + * + * @param count The allowed number of transformation configurations. + */ + final case class AllowedNumberOfConfigurations(count: Int) extends TenseiLicenseMessages + + /** + * Reports the allowed number of cronjobs. + * + * @param count The allowed number of cronjobs. + */ + final case class AllowedNumberOfCronjobs(count: Int) extends TenseiLicenseMessages + + /** + * Reports the allowed number of triggers. + * + * @param count The allowed number of triggers. + */ + final case class AllowedNumberOfTriggers(count: Int) extends TenseiLicenseMessages + + /** + * Reports the allowed number of users. + * + * @param count The allowed number of users. + */ + final case class AllowedNumberOfUsers(count: Int) extends TenseiLicenseMessages + + /** + * Reports the period in which the license will expire. + * + * @param period A period holding the amount of time in which the license will expire. + */ + final case class LicenseExpiresIn(period: Period) extends TenseiLicenseMessages + + /** + * Reports the license meta data. + * + * @param id The unique id of the license. + * @param licensee The name of the licensee (usually a company name). + * @param period A period holding the amount of time in which the license will expire. + */ + final case class LicenseMetaData(id: String, licensee: String, period: Period) + extends TenseiLicenseMessages + + /** + * Reports the entities information of the installed license. + * + * @param agents Number of agents. + * @param configurations Number of configurations. + * @param users Number of users. + * @param cronjobs Number of cronjobs. + * @param trigger Number of trigger. + */ + final case class LicenseEntitiesData( + agents: Int, + configurations: Int, + users: Int, + cronjobs: Int, + trigger: Int + ) extends TenseiLicenseMessages + + /** + * This message indicates that there is no license installed on the server. + */ + case object NoLicenseInstalled extends TenseiLicenseMessages + + /** + * Tell the server to report back the allowed number of agents. + */ + case object ReportAllowedNumberOfAgents extends TenseiLicenseMessages + + /** + * Tell the server to report back the allowed number of transformation configurations. + */ + case object ReportAllowedNumberOfConfigurations extends TenseiLicenseMessages + + /** + * Tell the server to report back the allowed number of cronjobs. + */ + case object ReportAllowedNumberOfCronjobs extends TenseiLicenseMessages + + /** + * Tell the server to report back the allowed number of triggers. + */ + case object ReportAllowedNumberOfTriggers extends TenseiLicenseMessages + + /** + * Tell the server to report back the allowed number of users. + */ + case object ReportAllowedNumberOfUsers extends TenseiLicenseMessages + + /** + * Tell the server to report back the expiration period of the license. + */ + case object ReportLicenseExpirationPeriod extends TenseiLicenseMessages + + /** + * Tell the server to report back the license meta data. + */ + case object ReportLicenseMetaData extends TenseiLicenseMessages + + /** + * Tell the server to report back the entity information of the license. + * Entities: Number of agents, configurations, users, cronjobs, trigger + */ + case object ReportLicenseEntitiesData extends TenseiLicenseMessages + + /** + * Update the currently used license with the one wrapped into this message. + * + * @param encodedLicense A string holding the encoded license. + */ + final case class UpdateLicense(encodedLicense: String) extends TenseiLicenseMessages + + /** + * Reports the result of a `UpdateLicense` action. + * + * @param message Holds a message that indicates either success of failure. + */ + final case class UpdateLicenseResult(message: String \/ String) extends TenseiLicenseMessages + + /** + * Validate the given encoded license. + * + * @param encodedLicense A string holding the encoded license. + */ + final case class ValidateLicense(encodedLicense: String) extends TenseiLicenseMessages + + /** + * Reports the result of a `ValidateLicense` action. + * + * @param result The details of the validation. + */ + final case class ValidateLicenseResult(result: LicenseValidationResult) + extends TenseiLicenseMessages + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/TransformationDataType.scala b/src/main/scala/com/wegtam/tensei/adt/TransformationDataType.scala new file mode 100644 index 0000000..713976b --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/TransformationDataType.scala @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +object TransformationDataType { + + private val legalClasses = List( + classOf[String], + classOf[java.lang.Integer], + classOf[java.lang.Long], + classOf[java.lang.Short], + classOf[java.lang.Byte], + classOf[java.lang.Character], + classOf[java.lang.Float], + classOf[java.lang.Double], + classOf[java.lang.Boolean], + classOf[Array[Byte]] + ) + + def isValidDataType(klass: Class[_]): Boolean = legalClasses.contains(klass) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/TransformationDescription.scala b/src/main/scala/com/wegtam/tensei/adt/TransformationDescription.scala new file mode 100644 index 0000000..4e69776 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/TransformationDescription.scala @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +/** + * Container for the description of an actual transformation for a piece of data. + * + * @param transformerClassName The name of the class that implements the transformer. + * @param options The options for the transformer. + */ +final case class TransformationDescription(transformerClassName: String, + options: TransformerOptions) + +object TransformationDescription { + + implicit def TransformationDescriptionCodecJson: CodecJson[TransformationDescription] = + CodecJson( + (t: TransformationDescription) ⇒ + ("options" := t.options) ->: + ("transformerClassName" := t.transformerClassName.toString) ->: + jEmptyObject, + c ⇒ + for { + transformerClassName ← (c --\ "transformerClassName").as[String] + options ← (c --\ "options").as[TransformerOptions] + } yield TransformationDescription(transformerClassName, options) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/adt/TransformerOptions.scala b/src/main/scala/com/wegtam/tensei/adt/TransformerOptions.scala new file mode 100644 index 0000000..0790d03 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/adt/TransformerOptions.scala @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import scala.language.existentials + +/** + * A case class to wrap around the standard options for a transformer. + * + * @param srcType The data type of the source data + * @param dstType The data type of the target data + * @param params A list of tuples that hold the parameters for the transformation actor. + */ +final case class TransformerOptions( + srcType: Class[_], + dstType: Class[_], + params: List[(String, String)] = List() +) { + + require(TransformationDataType.isValidDataType(srcType), s"Illegal source data type: $srcType") + require(TransformationDataType.isValidDataType(dstType), s"Illegal target data type: $dstType") + +} + +object TransformerOptions { + + // Codec for encoding and decoding a whole `TransformerOptions` object. + implicit def TransformerOptionsCodecJson: CodecJson[TransformerOptions] = + CodecJson( + (o: TransformerOptions) ⇒ + ("params" := o.params) ->: + ("dstType" := jString(o.dstType.getCanonicalName)) ->: + ("srcType" := jString(o.srcType.getCanonicalName)) ->: + jEmptyObject, + c ⇒ + for { + srcType ← (c --\ "srcType").as[String] + dstType ← (c --\ "dstType").as[String] + params ← (c --\ "params").as[List[(String, String)]] + } yield TransformerOptions(Class.forName(srcType), Class.forName(dstType), params) + ) + +} diff --git a/src/main/scala/com/wegtam/tensei/agent/ParserState.scala b/src/main/scala/com/wegtam/tensei/agent/ParserState.scala new file mode 100644 index 0000000..492793f --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/agent/ParserState.scala @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +/** + * This sealed trait describes all possible states of a parser on an agent system. + */ +sealed trait ParserState { + override def toString: String = this match { + case ParserState.Idle ⇒ "Idle" + case ParserState.ValidatingSyntax ⇒ "ValidatingSyntax" + case ParserState.ValidatingAccess ⇒ "ValidatingAccess" + case ParserState.ValidatingChecksums ⇒ "ValidatingChecksums" + case ParserState.PreparingSourceData ⇒ "PreparingSourceData" + case ParserState.InitializingSubParsers ⇒ "InitializingSubParsers" + case ParserState.Parsing ⇒ "Parsing" + } +} + +object ParserState { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def ParserStateCodecJson: CodecJson[ParserState] = + CodecJson( + (s: ParserState) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "Idle" ⇒ Idle + case "ValidatingSyntax" ⇒ ValidatingSyntax + case "ValidatingAccess" ⇒ ValidatingAccess + case "ValidatingChecksums" ⇒ ValidatingChecksums + case "PreparingSourceData" ⇒ PreparingSourceData + case "InitializingSubParsers" ⇒ InitializingSubParsers + case "Parsing" ⇒ Parsing + case e: String ⇒ throw new IllegalArgumentException(s"Unknown ParserState: '$e'!") + } + ) + + case object Idle extends ParserState + + case object ValidatingSyntax extends ParserState + + case object ValidatingAccess extends ParserState + + case object ValidatingChecksums extends ParserState + + case object PreparingSourceData extends ParserState + + case object InitializingSubParsers extends ParserState + + case object Parsing extends ParserState + +} diff --git a/src/main/scala/com/wegtam/tensei/agent/ProcessorState.scala b/src/main/scala/com/wegtam/tensei/agent/ProcessorState.scala new file mode 100644 index 0000000..a404997 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/agent/ProcessorState.scala @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +/** + * This sealed trait describes all possible states of a processor on an agent system. + */ +sealed trait ProcessorState { + override def toString: String = this match { + case ProcessorState.Idle ⇒ "Idle" + case ProcessorState.Sorting ⇒ "Sorting" + case ProcessorState.Processing ⇒ "Processing" + case ProcessorState.WaitingForWriterClosing ⇒ "WaitingForWriterClosing" + } +} + +object ProcessorState { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def ProcessorStateCodecJson: CodecJson[ProcessorState] = + CodecJson( + (s: ProcessorState) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "Idle" ⇒ Idle + case "Sorting" ⇒ Sorting + case "Processing" ⇒ Processing + case "WaitingForWriterClosing" ⇒ WaitingForWriterClosing + case e: String ⇒ throw new IllegalArgumentException(s"Unknown ProcessorState: '$e'!") + } + ) + + case object Idle extends ProcessorState + + case object Sorting extends ProcessorState + + case object Processing extends ProcessorState + + case object WaitingForWriterClosing extends ProcessorState + +} diff --git a/src/main/scala/com/wegtam/tensei/agent/TenseiAgentState.scala b/src/main/scala/com/wegtam/tensei/agent/TenseiAgentState.scala new file mode 100644 index 0000000..59fdbd4 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/agent/TenseiAgentState.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +/** + * This sealed trait describes all possible states of the `TenseiAgent` which leads + */ +sealed trait TenseiAgentState { + override def toString: String = this match { + case TenseiAgentState.Aborting ⇒ "Aborting" + case TenseiAgentState.CleaningUp ⇒ "CleaningUp" + case TenseiAgentState.Idle ⇒ "Idle" + case TenseiAgentState.InitializingResources ⇒ "InitializingResources" + case TenseiAgentState.Working ⇒ "Working" + } +} + +object TenseiAgentState { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def TenseiAgentStateCodecJson: CodecJson[TenseiAgentState] = + CodecJson( + (s: TenseiAgentState) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "Aborting" ⇒ Aborting + case "CleaningUp" ⇒ CleaningUp + case "Idle" ⇒ Idle + case "InitializingResources" ⇒ InitializingResources + case "Working" ⇒ Working + case e: String ⇒ throw new IllegalArgumentException(s"Unknown TenseiAgentState: '$e'!") + } + ) + + case object Aborting extends TenseiAgentState + + case object CleaningUp extends TenseiAgentState + + case object Idle extends TenseiAgentState + + case object InitializingResources extends TenseiAgentState + + case object Working extends TenseiAgentState + +} diff --git a/src/main/scala/com/wegtam/tensei/security/CryptoHelpers.scala b/src/main/scala/com/wegtam/tensei/security/CryptoHelpers.scala new file mode 100644 index 0000000..0ab4854 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/security/CryptoHelpers.scala @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.security + +import java.math.BigInteger +import java.security.interfaces.{ RSAPrivateKey, RSAPublicKey } +import java.security.spec.{ RSAPrivateKeySpec, RSAPublicKeySpec } +import java.security._ +import java.util.Base64 +import javax.crypto.spec.{ IvParameterSpec, SecretKeySpec } +import javax.crypto.{ Cipher, KeyGenerator, NoSuchPaddingException, SecretKey } + +import scalaz._ +import Scalaz._ + +/** + * This trait provides several helper functions for cryptography related tasks. + */ +trait CryptoHelpers { + // The default key length for AES keys. + val DEFAULT_AES_KEY_LENGTH = 128 + // The default variant of the AES cipher. + val DEFAULT_AES_VARIANT = "AES/CBC/PKCS5PADDING" + // The default key length for RSA keys. + val DEFAULT_RSA_KEY_LENGTH = 2048 + // The default variant of the RSA cipher. + val DEFAULT_RSA_VARIANT = "RSA/ECB/PKCS1Padding" + // The algorithm that is used to generate signatures. + val SIGNATURE_ALGORITHM = "SHA512withRSA" + + /** + * Return a secure random number generator. + * + * @return A seeded secure random number generator. + */ + protected def getRandomNumberGenerator: SecureRandom = + CryptoHelpers.randomNumberGenerator + + /** + * Decrypt the given data using the specified cipher and key. + * + * @param source The data that should be decrypted. + * @param cipher The cipher that should be used. + * @param key The key that should be used. + * @return Either an array of bytes holding the decrypted data or an exception. + */ + def decrypt(source: Array[Byte], cipher: Cipher, key: Key): Throwable \/ Array[Byte] = + decrypt(source, cipher, key, None) + + /** + * Decrypt the given data using the specified cipher, key and optional init vector. + * + * @param source The data that should be decrypted. + * @param cipher The cipher that should be used. + * @param key The key that should be used. + * @param iv An optional init vector which is needed if you want to decrypt AES encrypted data. + * @return Either an array of bytes holding the decrypted data or an exception. + */ + def decrypt(source: Array[Byte], + cipher: Cipher, + key: Key, + iv: Option[Array[Byte]]): Throwable \/ Array[Byte] = + try { + iv.fold(cipher.init(Cipher.DECRYPT_MODE, key))( + i ⇒ cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(i)) + ) + cipher.doFinal(source).right + } catch { + case e: Throwable ⇒ e.left + } + + /** + * Encrypt the given source (an array of bytes) using the provided cipher and key. + * The encrypted data is returned in base 64 format. + * + * @param source The data to encrypt. + * @param cipher The cipher that should be used. + * @param key The key that should be used to encrypt the data. + * @return Either an array holding the base 64 encoded encrypted message or an exception. + */ + def encrypt(source: Array[Byte], cipher: Cipher, key: Key): Throwable \/ Array[Byte] = + encrypt(source, cipher, key, None) + + /** + * Encrypt the given source (an array of bytes) using the provided cipher and key. + * The encrypted data is returned in base 64 format. + * + * @param source The data to encrypt. + * @param cipher The cipher that should be used. + * @param key The key that should be used to encrypt the data. + * @param iv An optional init vector which is needed if you want to decrypt AES encrypted data. + * @return Either an array holding the base 64 encoded encrypted message or an exception. + */ + def encrypt(source: Array[Byte], + cipher: Cipher, + key: Key, + iv: Option[Array[Byte]] = None): Throwable \/ Array[Byte] = + try { + iv.fold(cipher.init(Cipher.ENCRYPT_MODE, key))( + i ⇒ cipher.init(Cipher.ENCRYPT_MODE, key, new IvParameterSpec(i)) + ) + val encryptedSource = cipher.doFinal(source) + Base64.getEncoder.encode(encryptedSource).right + } catch { + case e: Throwable ⇒ e.left + } + + /** + * Generate an AES key and an init vector using `DEFAULT_AES_KEY_LENGTH`. + * + * @return A tuple holding the generated key and the init vector. + */ + def generateAESKeyAndIV(): (SecretKey, Array[Byte]) = + generateAESKeyAndIV(DEFAULT_AES_KEY_LENGTH) + + /** + * Generate an AES key and an init vector. + * + * @param keySize The desired key size. + * @return A tuple holding the generated key and the init vector. + */ + def generateAESKeyAndIV(keySize: Int): (SecretKey, Array[Byte]) = { + val keyGenerator = KeyGenerator.getInstance("AES") + keyGenerator.init(keySize) + val secretKey = keyGenerator.generateKey() + + val secureRandom = getRandomNumberGenerator + val initVector = new Array[Byte](keySize / 8) // The IV must have the same number of bits as the key. + secureRandom.nextBytes(initVector) + + (secretKey, initVector) + } + + /** + * Regenerate an AES key from the given bytes. + * + * @param keyBytes The key's bytes (produced via `getEncoded`) + * @return The generated secret key. + */ + def generateAESKeyFromParameters(keyBytes: Array[Byte]): SecretKey = + new SecretKeySpec(keyBytes, "AES") + + /** + * Generate a RSA key pair using `DEFAULT_RSA_KEY_LENGTH`. + * + * @return A RSA key pair. + */ + def generateRSAKeyPair(): KeyPair = generateRSAKeyPair(DEFAULT_RSA_KEY_LENGTH) + + /** + * Generate a RSA key pair using the specified key size. + * + * @param keySize The desired key size. + * @return A RSA key pair. + */ + def generateRSAKeyPair(keySize: Int): KeyPair = { + val keyGenerator = KeyPairGenerator.getInstance("RSA") + val secureRandom = getRandomNumberGenerator + keyGenerator.initialize(keySize, secureRandom) + + keyGenerator.generateKeyPair() + } + + /** + * Generate a RSA key using the specified parameters. + * + * @param modulus The modulus for the key. + * @param exponent The exponend for the key. + * @param keyType The type of the key that should be returned. Must be either `classOf[RSAPrivateKey]` or `classOf[RSAPublicKey]`. + * @return The generated key of the desired type. + */ + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + def generateRSAKeyFromParameters(modulus: Array[Byte], + exponent: Array[Byte], + keyType: Class[_]): Key = { + // We need to create stable classifiers for the pattern matching below. + val PRIVATE = classOf[RSAPrivateKey] + val PUBLIC = classOf[RSAPublicKey] + + val keyFactory = KeyFactory.getInstance("RSA") + val mod = new BigInteger(modulus) + val exp = new BigInteger(exponent) + + keyType match { + case PRIVATE ⇒ + val keySpec = new RSAPrivateKeySpec(mod, exp) + keyFactory.generatePrivate(keySpec) + case PUBLIC ⇒ + val keySpec = new RSAPublicKeySpec(mod, exp) + keyFactory.generatePublic(keySpec) + case _ ⇒ throw new IllegalArgumentException(s"Unknown RSA key type: $keyType!") + } + } + + /** + * Get an AES cipher that can be used for encryption or decryption of data. + * + * @return An AES cipher. + */ + @throws[NoSuchAlgorithmException] + @throws[NoSuchPaddingException] + def getAESCipher: Cipher = Cipher.getInstance(DEFAULT_AES_VARIANT) + + /** + * Get a RSA cipher that can be used for encryption or decryption of data. + * + * @return A RSA cipher. + */ + @throws[NoSuchAlgorithmException] + @throws[NoSuchPaddingException] + def getRSACipher: Cipher = Cipher.getInstance(DEFAULT_RSA_VARIANT) + + /** + * Generate a signature for the source data using `SIGNATURE_ALGORITHM`. + * The given private key should be feasable for the signature algorithm! + * + * @param source The data to be signed. + * @param key A private key. + * @return Either the base 64 encoded signature or an exception. + */ + def sign(source: Array[Byte], key: PrivateKey): Throwable \/ Array[Byte] = + try { + val signatureProvider = Signature.getInstance(SIGNATURE_ALGORITHM) + signatureProvider.initSign(key, getRandomNumberGenerator) + signatureProvider.update(source) + val signature = signatureProvider.sign() + Base64.getEncoder.encode(signature).right + } catch { + case e: Throwable ⇒ e.left + } + + /** + * Validate the given signature. + * This method may produce exceptions. + * + * @param message The message that should match the signature. + * @param signature The signature. + * @param key The key that is used to verify the signature. + * @return Either `true` if the signature is valid or `false` otherwise. + * @todo Fix the scaladoc configuration somehow to avoid the `Could not find any member to link for "..."` exception! + */ + @throws[InvalidKeyException]("The provided key is invalid!") + @throws[NoSuchAlgorithmException]( + "No Provider supports a Signature implementation for the specified algorithm!" + ) + @throws[SignatureException]( + "The signature object is not initialized properly, the passed-in signature is improperly encoded or of the wrong type, the signature algorithm is unable to process the input data provided, etc." + ) + def validate(message: Array[Byte], signature: Array[Byte], key: PublicKey): Boolean = { + val signatureProvider = Signature.getInstance(SIGNATURE_ALGORITHM) + signatureProvider.initVerify(key) + signatureProvider.update(message) + signatureProvider.verify(signature) + } +} + +object CryptoHelpers { + + // A secure random number generator according to NIST recommendations. + private final val randomNumberGenerator: SecureRandom = { + val secureRandom = new SecureRandom() // Prefer this because SecureRandom.getInstanceStrong blocks a lot on systems with low entropy. + secureRandom.setSeed(secureRandom.generateSeed(55)) // NIST SP800-90A recommends a seed length of 440 bits (i.e. 55 bytes) + secureRandom + } + +} diff --git a/src/main/scala/com/wegtam/tensei/server/ChefDeCuisineState.scala b/src/main/scala/com/wegtam/tensei/server/ChefDeCuisineState.scala new file mode 100644 index 0000000..cc52000 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/server/ChefDeCuisineState.scala @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server + +import argonaut._, Argonaut._ + +/** + * This sealed trait describes all possible states of the chef de cuisine. + */ +sealed trait ChefDeCuisineState { + override def toString: String = this match { + case ChefDeCuisineState.Booting ⇒ "Booting" + case ChefDeCuisineState.Initializing ⇒ "Initializing" + case ChefDeCuisineState.Running ⇒ "Running" + } +} + +object ChefDeCuisineState { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def ChefDeCuisineStateCodecJson: CodecJson[ChefDeCuisineState] = + CodecJson( + (s: ChefDeCuisineState) ⇒ jString(s.toString), + c ⇒ + for { + s ← c.as[String] + } yield + s match { + case "Booting" ⇒ Booting + case "Initializing" ⇒ Initializing + case "Running" ⇒ Running + case e: String ⇒ + throw new IllegalArgumentException(s"Unknown ChefDeCuisineState: '$e'!") + } + ) + + case object Booting extends ChefDeCuisineState + + case object Initializing extends ChefDeCuisineState + + case object Running extends ChefDeCuisineState + +} diff --git a/src/main/scala/com/wegtam/tensei/server/ServerMessages.scala b/src/main/scala/com/wegtam/tensei/server/ServerMessages.scala new file mode 100644 index 0000000..352ef44 --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/server/ServerMessages.scala @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server + +import com.wegtam.tensei.adt.{ AgentInformation, AgentStartTransformationMessage, StatusMessage } + +import scalaz._ + +/** + * A sealed trait containing the server messages that should be available accross the cluster. + */ +sealed trait ServerMessages + +/** + * Container object holding the server messages the keep the namespace somewhat clean. + */ +object ServerMessages { + + /** + * Notify the target actor to return it's collected agents informations. + */ + case object ReportAgentsInformations extends ServerMessages + + /** + * Holds the collected agents informations. + * + * @param agents A map containing the informations for each agent mapped by the agent id. + */ + final case class ReportAgentsInformationsResponse(agents: Map[String, AgentInformation]) + extends ServerMessages + + /** + * Instruct the receiver to start a given transformation. + * + * @param message An option to an agent start transformation message that contains all necessary informations. + */ + final case class StartTransformationConfiguration( + message: Option[AgentStartTransformationMessage] + ) extends ServerMessages + + /** + * The response from the server for a start transformation configuration message. + * + * @param statusMessage A return message holding either a status message or an error message. + */ + final case class StartTransformationConfigurationResponse(statusMessage: StatusMessage \/ String, + uniqueIdentifier: Option[String]) + extends ServerMessages + +} diff --git a/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterMessages.scala b/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterMessages.scala new file mode 100644 index 0000000..44e5f8f --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterMessages.scala @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server.suggesters + +import akka.actor.ActorRef +import com.wegtam.tensei.adt.{ Cookbook, StatusMessage } + +/** + * The messages that can passed to and received from a mapping suggester. + */ +sealed trait MappingSuggesterMessages + +object MappingSuggesterMessages { + + /** + * If an error occurs, this message is used to return the error details. + * + * @param error An error message. + */ + final case class MappingSuggesterErrorMessage(error: StatusMessage) + extends MappingSuggesterMessages + + /** + * Suggest and return a mapping for the given cookbook. + * + * @param cookbook The cookbook that holds the information needed for the mapping creation. + * @param mode The mode of the mapping suggester (`MappingSuggesterModes`) which defaults to `Simple`. + * @param answerTo An option to an actor ref that should receive the answer instead of the sender. + */ + final case class SuggestMapping(cookbook: Cookbook, + mode: MappingSuggesterModes = MappingSuggesterModes.Simple, + answerTo: Option[ActorRef] = None) + extends MappingSuggesterMessages + + /** + * This message holds the cookbook containing the suggested mapping. + * + * @param cookbook The cookbook holding the mapping. + * @param mode The mode that was used to create the mapping. + */ + final case class SuggestedMapping(cookbook: Cookbook, mode: MappingSuggesterModes) + extends MappingSuggesterMessages + +} diff --git a/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModes.scala b/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModes.scala new file mode 100644 index 0000000..d64e06b --- /dev/null +++ b/src/main/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModes.scala @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server.suggesters + +import argonaut._, Argonaut._ + +/** + * The possible modes for a mapping suggester. + */ +sealed trait MappingSuggesterModes { + override def toString: String = this match { + case MappingSuggesterModes.Simple ⇒ "Simple" + case MappingSuggesterModes.SimpleWithTransformers ⇒ "SimpleWithTransformers" + case MappingSuggesterModes.SimpleSemantics ⇒ "SimpleSemantics" + case MappingSuggesterModes.SimpleSemanticsWithTransformers ⇒ "SimpleSemanticsWithTransformers" + case MappingSuggesterModes.AdvancedSemantics ⇒ "AdvancedSemantics" + } +} + +/** + * Container object to keep the namespace clean. + */ +object MappingSuggesterModes { + @SuppressWarnings(Array("org.wartremover.warts.Throw")) + implicit def MappingSuggesterModesCodecJson: CodecJson[MappingSuggesterModes] = + CodecJson( + (m: MappingSuggesterModes) ⇒ jString(m.toString), + c ⇒ + for { + m ← c.as[String] + } yield + m match { + case "Simple" ⇒ Simple + case "SimpleWithTransformers" ⇒ SimpleWithTransformers + case "SimpleSemantics" ⇒ SimpleSemantics + case "SimpleSemanticsWithTransformers" ⇒ SimpleSemanticsWithTransformers + case "AdvancedSemantics" ⇒ AdvancedSemantics + case e: String ⇒ + throw new IllegalArgumentException(s"Unknown MappingSuggesterMode: '$e'!") + } + ) + + case object Simple extends MappingSuggesterModes + + case object SimpleWithTransformers extends MappingSuggesterModes + + case object SimpleSemantics extends MappingSuggesterModes + + case object SimpleSemanticsWithTransformers extends MappingSuggesterModes + + case object AdvancedSemantics extends MappingSuggesterModes + +} diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf new file mode 100644 index 0000000..6bd9269 --- /dev/null +++ b/src/test/resources/application.conf @@ -0,0 +1,25 @@ +# A file for testing our settings extension. + +tensei { + # Global configuration goes here... + + test-option = "I am a test option..." + + agents { + # Configuration options for all agents... + + ask-timeout = 5 seconds + } + + frontend { + # Frontend configuration... + + test-option = 42 + } + + server { + # Server side configuration... + + test-option = 3.14 + } +} \ No newline at end of file diff --git a/src/test/scala/com/wegtam/tensei/DefaultSpec.scala b/src/test/scala/com/wegtam/tensei/DefaultSpec.scala new file mode 100644 index 0000000..a75a0a0 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/DefaultSpec.scala @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei + +import org.scalatest.{ FunSpec, MustMatchers } + +abstract class DefaultSpec extends FunSpec with MustMatchers {} diff --git a/src/test/scala/com/wegtam/tensei/adt/AgentAuthorizationStateTest.scala b/src/test/scala/com/wegtam/tensei/adt/AgentAuthorizationStateTest.scala new file mode 100644 index 0000000..eae9f4b --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/AgentAuthorizationStateTest.scala @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class AgentAuthorizationStateTest extends DefaultSpec { + describe("AgentAuthorizationState") { + describe("CodecJson") { + describe("decode") { + it("must decode all known states correctly") { + Parse.decodeOption[AgentAuthorizationState](""""Connected"""").get must be( + AgentAuthorizationState.Connected + ) + Parse.decodeOption[AgentAuthorizationState](""""Disconnected"""").get must be( + AgentAuthorizationState.Disconnected + ) + Parse.decodeOption[AgentAuthorizationState](""""Unauthorized"""").get must be( + AgentAuthorizationState.Unauthorized + ) + } + + it("must throw an IllegalArgumentException upon an unknown state") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[AgentAuthorizationState](""""SomeUnknownState"""") + ) + } + } + + describe("encode") { + it("must encode all known states correctly") { + AgentAuthorizationState.Connected.asJson.nospaces must be(""""Connected"""") + AgentAuthorizationState.Disconnected.asJson.nospaces must be(""""Disconnected"""") + AgentAuthorizationState.Unauthorized.asJson.nospaces must be(""""Unauthorized"""") + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/AgentInformationTest.scala b/src/test/scala/com/wegtam/tensei/adt/AgentInformationTest.scala new file mode 100644 index 0000000..44839e8 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/AgentInformationTest.scala @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec +import com.wegtam.tensei.agent.{ ParserState, ProcessorState, TenseiAgentState } + +class AgentInformationTest extends DefaultSpec { + describe("AgentInformation") { + describe("CodecJson") { + describe("decode") { + it("must decode proper json") { + val json = + """ + |{ + | "path":"some-actor-path", + | "auth":"Connected", + | "id":"AGENT-ID", + | "updated":1417704004071, + | "workingState":{ + | "state":"Working", + | "parser":"Parsing", + | "currentTransformationId":null, + | "id":"AGENT-ID", + | "processor":"Idle", + | "runtime":{ + | "agent1.example.com": { + | "free":111101592, + | "max":1900019712, + | "total":128974848, + | "processors":1, + | "load":null + | } + | } + | } + |} + """.stripMargin + + val expected = new AgentInformation( + id = "AGENT-ID", + auth = AgentAuthorizationState.Connected, + path = "some-actor-path", + lastUpdated = 1417704004071L, + workingState = Option( + new AgentWorkingState( + id = "AGENT-ID", + state = TenseiAgentState.Working, + parser = ParserState.Parsing, + processor = ProcessorState.Idle, + runtimeStats = Map( + "agent1.example.com" → new RuntimeStats(freeMemory = 111101592, + maxMemory = 1900019712, + totalMemory = 128974848) + ) + ) + ) + ) + + Parse.decodeOption[AgentInformation](json).get must be(expected) + } + } + + describe("encode") { + it("must encode to proper json") { + val info = new AgentInformation( + id = "AGENT-ID", + auth = AgentAuthorizationState.Connected, + path = "some-actor-path", + lastUpdated = 1417704004072L, + workingState = Option( + new AgentWorkingState( + id = "AGENT-ID", + state = TenseiAgentState.Working, + parser = ParserState.Parsing, + processor = ProcessorState.Idle, + runtimeStats = Map( + "agent42.example.com" → new RuntimeStats(freeMemory = 111101592, + maxMemory = 1900019712, + totalMemory = 128974848) + ) + ) + ) + ) + + val expected = + """{"path":"some-actor-path","auth":"Connected","id":"AGENT-ID","updated":1417704004072,"workingState":{"uniqueIdentifier":null,"state":"Working","parser":"Parsing","id":"AGENT-ID","processor":"Idle","runtime":{"agent42.example.com":{"processors":1,"load":null,"total":128974848,"max":1900019712,"free":111101592}}}}""" + + info.asJson.nospaces must be(expected) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/AgentStartTransformationMessageTest.scala b/src/test/scala/com/wegtam/tensei/adt/AgentStartTransformationMessageTest.scala new file mode 100644 index 0000000..290ef1c --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/AgentStartTransformationMessageTest.scala @@ -0,0 +1,445 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.DefaultSpec +import java.net.URI + +import com.wegtam.tensei.adt.Recipe.MapAllToAll + +import scalaz._ + +class AgentStartTransformationMessageTest extends DefaultSpec { + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val con1 = new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("MY-COOKBOOK", "SOURCE-01")), + Some("user01"), + Some("pw01"), + None, + None) + val con2 = new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("MY-COOKBOOK", "TARGET-ID")), + Some("user02"), + Some("pw02"), + None, + Some("de_DE")) + + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sourceElements, targetElements, List(t)) + val r = List(new Recipe("my-recipe", MapAllToAll, List(m))) + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + val as = new AgentStartTransformationMessage(List(con1), + con2, + Cookbook("MY-COOKBOOK", sources, target, r)) + + val expectedJson = + """ + |{ + | "target" : { + | "username" : "user02", + | "uri" : "http://www.example.com", + | "dfasdlRef" : { + | "cookbook-id" : "MY-COOKBOOK", + | "dfasdl-id" : "TARGET-ID" + | }, + | "checksum" : null, + | "password" : "pw02", + | "languageTag" : "de_DE" + | }, + | "sources" : [ + | { + | "username" : "user01", + | "uri" : "http://www.example.com", + | "dfasdlRef" : { + | "cookbook-id" : "MY-COOKBOOK", + | "dfasdl-id" : "SOURCE-01" + | }, + | "checksum" : null, + | "password" : "pw01", + | "languageTag" : null + | } + | ], + | "cookbook" : { + | "id" : "MY-COOKBOOK", + | "sources" : [ + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-01" + | }, + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-02" + | } + | ], + | "target" : { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "TARGET-ID" + | }, + | "recipes" : [ + | { + | "id" : "my-recipe", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "mappingKey" : null, + | "sources" : [ + | {"elementId": "source01", "dfasdlId": "DFASDL"}, + | {"elementId": "source02", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations" : [ + | + | ] + | } + | ] + | } + | ] + | } + | ,"uniqueIdentifier":null + |} + """.stripMargin + + as.asJson.nospaces mustEqual Parse.parseOption(expectedJson).get.nospaces + } + } + + describe("decode") { + it("must properly decode json to an object") { + val jsonString = + """ + |{ + | "target" : { + | "username" : "username02", + | "uri" : "http://www.example2.com", + | "dfasdlRef" : { + | "cookbook-id" : "ANOTHER-COOKBOOK", + | "dfasdl-id" : "TARGET-ID" + | }, + | "checksum" : null, + | "password" : "passw02" + | }, + | "sources" : [ + | { + | "username" : "username01", + | "uri" : "http://www.example.com", + | "dfasdlRef" : { + | "cookbook-id" : "ANOTHER-COOKBOOK", + | "dfasdl-id" : "SOURCE-02" + | }, + | "checksum" : null, + | "password" : "passw01", + | "languageTag" : "en_US" + | } + | ], + | "cookbook" : { + | "id" : "ANOTHER-COOKBOOK", + | "sources" : [ + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-01" + | }, + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-02" + | } + | ], + | "target" : { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "TARGET-ID" + | }, + | "recipes" : [ + | { + | "id" : "my-recipe2", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "sources" : [ + | {"elementId": "source01", "dfasdlId": "DFASDL"}, + | {"elementId": "source02", "dfasdlId": "DFASDL"} + | ], + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations": [] + | } + | ] + | } + | ] + | } + |} + """.stripMargin + + val con1 = + new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-02")), + Some("username01"), + Some("passw01"), + None, + Some("en_US")) + val con2 = + new ConnectionInformation(new URI("http://www.example2.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "TARGET-ID")), + Some("username02"), + Some("passw02"), + None) + + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sourceElements, targetElements, List(t)) + val r = List(new Recipe("my-recipe2", MapAllToAll, List(m))) + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val expected = + new AgentStartTransformationMessage(List(con1), + con2, + Cookbook("ANOTHER-COOKBOOK", sources, target, r)) + + Parse.decodeEither[AgentStartTransformationMessage](jsonString) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ success must be(expected) + } + } + } + } + + describe("hasChecksums") { + describe("with checksums") { + it("must return true") { + val con1 = + new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-01")), + Some("username01"), + Some("passw01"), + None) + val con2 = new ConnectionInformation( + new URI("http://www.example.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-02")), + Some("username01"), + Some("passw01"), + Some("7b6f0635cabc5c86934586fa2338b238530733e5581bba9bf8409e32fe7f0c8c") + ) + val con3 = + new ConnectionInformation(new URI("http://www.example2.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "TARGET-ID")), + Some("username02"), + Some("passw02"), + None) + + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sourceElements, targetElements, List(t)) + val r = List(new Recipe("my-recipe2", MapAllToAll, List(m))) + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val message = + new AgentStartTransformationMessage(List(con1, con2), + con3, + Cookbook("ANOTHER-COOKBOOK", sources, target, r)) + + message.hasChecksums must be(true) + } + } + + describe("without checksums") { + it("must return false") { + val con1 = + new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-02")), + Some("username01"), + Some("passw01"), + None) + val con2 = + new ConnectionInformation(new URI("http://www.example2.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-03")), + Some("username02"), + Some("passw02"), + None) + val con3 = + new ConnectionInformation(new URI("http://www.example2.com"), + Some(DFASDLReference("ANOTHER-COOKBOOK", "TARGET-ID")), + Some("username02"), + Some("passw02"), + None) + + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sourceElements, targetElements, List(t)) + val r = List(new Recipe("my-recipe2", MapAllToAll, List(m))) + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val message = + new AgentStartTransformationMessage(List(con1, con2), + con3, + Cookbook("ANOTHER-COOKBOOK", sources, target, r)) + + message.hasChecksums must be(false) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/AgentWorkingStateTest.scala b/src/test/scala/com/wegtam/tensei/adt/AgentWorkingStateTest.scala new file mode 100644 index 0000000..f58c200 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/AgentWorkingStateTest.scala @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec +import com.wegtam.tensei.agent.{ ParserState, ProcessorState, TenseiAgentState } + +class AgentWorkingStateTest extends DefaultSpec { + describe("AgentWorkingState") { + describe("CodecJson") { + describe("decode") { + describe("without current transformation id") { + it("must decode a proper json string") { + val json = + """ + |{ + | "id":"MY-AGENT", + | "state":"Working", + | "parser":"Parsing", + | "processor":"Idle", + | "runtime":{ + | "1": { + | "total":200, + | "max":100, + | "free":50, + | "processors":1, + | "load":null + | } + | } + |} + """.stripMargin + val expectedWorkingState = AgentWorkingState("MY-AGENT", + TenseiAgentState.Working, + ParserState.Parsing, + ProcessorState.Idle, + Map("1" → RuntimeStats(50, 100, 200)), + None) + + Parse.decodeOption[AgentWorkingState](json).get must be(expectedWorkingState) + } + } + + describe("with current transformation id") { + it("must decode a proper json string") { + val json = + """ + |{ + | "id":"MY-AGENT", + | "state":"Working", + | "parser":"Parsing", + | "processor":"Idle", + | "runtime":{ + | "1": { + | "total":200, + | "max":100, + | "free":50, + | "processors":1, + | "load":null + | } + | }, + | "uniqueIdentifier":"de305d54-75b4-431b-adb2-eb6b9e546013" + |} + """.stripMargin + val expectedWorkingState = AgentWorkingState( + "MY-AGENT", + TenseiAgentState.Working, + ParserState.Parsing, + ProcessorState.Idle, + Map("1" → RuntimeStats(50, 100, 200)), + Option("de305d54-75b4-431b-adb2-eb6b9e546013") + ) + + Parse.decodeOption[AgentWorkingState](json).get must be(expectedWorkingState) + } + } + } + + describe("encode") { + describe("without current transformation id") { + it("must encode to proper json") { + val state = AgentWorkingState("MY-AGENT", + TenseiAgentState.Working, + ParserState.Parsing, + ProcessorState.Idle, + Map("1" → RuntimeStats(50, 100, 200))) + val expectedJson = + """{"uniqueIdentifier":null,"state":"Working","parser":"Parsing","id":"MY-AGENT","processor":"Idle","runtime":{"1":{"processors":1,"load":null,"total":200,"max":100,"free":50}}}""" + state.asJson.nospaces must be(expectedJson) + } + } + + describe("with current transformation id") { + it("must encode to proper json") { + val state = AgentWorkingState( + "MY-AGENT", + TenseiAgentState.Working, + ParserState.Parsing, + ProcessorState.Idle, + Map("1" → RuntimeStats(50, 100, 200)), + Option("de305d54-75b4-431b-adb2-eb6b9e546013") + ) + val expectedJson = + """{"uniqueIdentifier":"de305d54-75b4-431b-adb2-eb6b9e546013","state":"Working","parser":"Parsing","id":"MY-AGENT","processor":"Idle","runtime":{"1":{"processors":1,"load":null,"total":200,"max":100,"free":50}}}""" + state.asJson.nospaces must be(expectedJson) + } + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/AtomicTransformationDescriptionTest.scala b/src/test/scala/com/wegtam/tensei/adt/AtomicTransformationDescriptionTest.scala new file mode 100644 index 0000000..fa74a2b --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/AtomicTransformationDescriptionTest.scala @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +import scalaz._ + +class AtomicTransformationDescriptionTest extends DefaultSpec { + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = + new AtomicTransformationDescription(ElementReference("MY-DFASDL-ID", "MY-SOURCE-ID"), + "com.example.transformers.foo", + o) + val expectedJson = + """{"element":{"elementId":"MY-SOURCE-ID","dfasdlId":"MY-DFASDL-ID"},"transformerClassName":"com.example.transformers.foo","options":{"srcType":"java.lang.String","dstType":"java.lang.Long","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}}""" + t.asJson.nospaces mustEqual expectedJson + } + } + + describe("decode") { + it("must properly decode json to an object") { + val params = List(("one", "1"), ("two", "Zweikommasechs"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[java.lang.Integer], classOf[java.lang.Long], params) + val expected = + new AtomicTransformationDescription(ElementReference("DFASDL-ID", "ANOTHER-SOURCE-ID"), + "com.example.transformers.bar", + o) + val jsonString = + """{"element":{"elementId":"ANOTHER-SOURCE-ID","dfasdlId":"DFASDL-ID"},"transformerClassName":"com.example.transformers.bar","options":{"srcType":"java.lang.Integer","dstType":"java.lang.Long","params":[["one","1"],["two","Zweikommasechs"],["three","3.14f"]]}}""" + Parse.decodeEither[AtomicTransformationDescription](jsonString) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + success must be(expected) + success.element must be(expected.element) + success.transformerClassName must be(expected.transformerClassName) + success.options must be(expected.options) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/ClusterConstants$Test.scala b/src/test/scala/com/wegtam/tensei/adt/ClusterConstants$Test.scala new file mode 100644 index 0000000..f4186af --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/ClusterConstants$Test.scala @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import com.wegtam.tensei.DefaultSpec + +/** + * We test our cluster role names because we need to be alerted if we ever change them! + */ +class ClusterConstants$Test extends DefaultSpec { + describe("the name for the system") { + val name = "tensei-system" + it(s"must be '$name'") { + ClusterConstants.systemName must be(name) + } + } + + describe("the top level actor name on an agent system") { + val name = "TenseiAgent" + it(s"must be '$name'") { + ClusterConstants.topLevelActorNameOnAgent must be(name) + } + } + + describe("the top level actor name on the server system") { + val name = "TenseiServer" + it(s"must be '$name'") { + ClusterConstants.topLevelActorNameOnServer must be(name) + } + } + + describe("the name for an agent node") { + val name = "agent" + it(s"must be '$name'") { + ClusterConstants.Roles.agent must be(name) + } + } + + describe("the name for a frontend node") { + val name = "frontend" + it(s"must be '$name'") { + ClusterConstants.Roles.frontend must be(name) + } + } + + describe("the name for a server node") { + val name = "server" + it(s"must be '$name'") { + ClusterConstants.Roles.server must be(name) + } + } + + describe("the name for a watchdog node") { + val name = "watchdog" + it(s"must be '$name'") { + ClusterConstants.Roles.watchdog must be(name) + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/ConnectionInformationTest.scala b/src/test/scala/com/wegtam/tensei/adt/ConnectionInformationTest.scala new file mode 100644 index 0000000..5330480 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/ConnectionInformationTest.scala @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import java.net.URI + +import argonaut.Argonaut._ +import argonaut._ +import com.wegtam.tensei.DefaultSpec + +class ConnectionInformationTest extends DefaultSpec { + + describe("JsonCodec") { + describe("encode") { + describe("with a dfasdl reference") { + it("must properly encode an object to json") { + val c = new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("MY-COOKBOOK", "MY-DFASDL")), + Some("testuser"), + Some("testpw"), + None, + Some("en_US")) + val expectedJson = + """ + |{ + | "username" : "testuser", + | "uri" : "http://www.example.com", + | "dfasdlRef" : { + | "cookbook-id" : "MY-COOKBOOK", + | "dfasdl-id" : "MY-DFASDL" + | }, + | "checksum" : null, + | "password" : "testpw", + | "languageTag" : "en_US" + |} + """.stripMargin + + c.asJson.nospaces mustEqual Parse.parseOption(expectedJson).get.nospaces + } + } + + describe("without a dfasdl reference") { + it("must properly encode an object to json") { + val c = new ConnectionInformation(new URI("http://www.example.com"), + None, + Some("testuser"), + Some("testpw"), + None, + None) + val expectedJson = + """{"languageTag":null,"username":"testuser","uri":"http://www.example.com","dfasdlRef":null,"checksum":null,"password":"testpw"}""" + c.asJson.nospaces mustEqual expectedJson + } + } + } + + describe("decode") { + describe("with a dfasdl reference") { + it("must properly decode json to an object") { + val jsonString = + """ + |{ + | "username" : "user01", + | "uri" : "http://www.example.com", + | "dfasdlRef" : { + | "cookbook-id" : "MY-COOKBOOK", + | "dfasdl-id" : "MY-DFASDL" + | }, + | "checksum" : null, + | "password" : "pw01", + | "languageTag" : "de_DE" + |} + """.stripMargin + val expected = + new ConnectionInformation(new URI("http://www.example.com"), + Some(DFASDLReference("MY-COOKBOOK", "MY-DFASDL")), + Some("user01"), + Some("pw01"), + None, + Some("de_DE")) + val decoded: Option[ConnectionInformation] = + Parse.decodeOption[ConnectionInformation](jsonString) + decoded.isDefined must be(true) + decoded.get must be(expected) + decoded.get.uri must be(expected.uri) + decoded.get.dfasdlRef must be(expected.dfasdlRef) + decoded.get.username must be(expected.username) + decoded.get.password must be(expected.password) + decoded.get.checksum must be(expected.checksum) + } + } + + describe("without a dfasdl reference") { + it("must properly decode json to an object") { + val jsonString = + """{"dfasdlRef":null,"languageTag":null,"username":"user01","uri":"http://www.example.com","checksum":null,"password":"pw01"}""" + val expected = new ConnectionInformation(new URI("http://www.example.com"), + None, + Some("user01"), + Some("pw01"), + None, + None) + val decoded: Option[ConnectionInformation] = + Parse.decodeOption[ConnectionInformation](jsonString) + decoded.isDefined must be(true) + decoded.get must be(expected) + decoded.get.uri must be(expected.uri) + decoded.get.dfasdlRef must be(expected.dfasdlRef) + decoded.get.username must be(expected.username) + decoded.get.password must be(expected.password) + decoded.get.checksum must be(expected.checksum) + } + } + } + } + +} diff --git a/src/test/scala/com/wegtam/tensei/adt/CookbookTest.scala b/src/test/scala/com/wegtam/tensei/adt/CookbookTest.scala new file mode 100644 index 0000000..03dd365 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/CookbookTest.scala @@ -0,0 +1,581 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +import scalaz._ + +class CookbookTest extends DefaultSpec { + describe("Constraints") { + describe("id") { + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val params1 = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o1 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params1) + val t1 = new TransformationDescription("com.example.transformers.foo", o1) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m1 = new MappingTransformation(sourceElements, targetElements, List(t1)) + val recipe1 = Recipe.createAllToAllRecipe("RECIPE-01", List(m1)) + + val params2 = List(("one", "1"), ("two", "2.71f"), ("three", "3.00")) + val o2 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params2) + val t2 = new TransformationDescription("com.example.transformers.bar", o2) + val sourceElements2 = List( + ElementReference("DFASDL", "source04"), + ElementReference("DFASDL", "source05") + ) + val targetElements2 = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m2 = new MappingTransformation(sourceElements2, targetElements2, List(t2)) + val recipe2 = Recipe.createAllToAllRecipe("RECIPE-02", List(m2)) + + it("must not be empty") { + an[IllegalArgumentException] must be thrownBy Cookbook("", + sources, + target, + List(recipe1, recipe2)) + } + + it("must not be null") { + an[IllegalArgumentException] must be thrownBy Cookbook(null, + sources, + target, + List(recipe1, recipe2)) + } + } + } + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val params1 = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o1 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params1) + val t1 = new TransformationDescription("com.example.transformers.foo", o1) + val sourceElements = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m1 = new MappingTransformation(sourceElements, targetElements, List(t1)) + val recipe1 = Recipe.createAllToAllRecipe("RECIPE-01", List(m1)) + + val params2 = List(("one", "1"), ("two", "2.71f"), ("three", "3.00")) + val o2 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params2) + val t2 = new TransformationDescription("com.example.transformers.bar", o2) + val sourceElements2 = List( + ElementReference("DFASDL", "source04"), + ElementReference("DFASDL", "source05") + ) + val targetElements2 = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m2 = new MappingTransformation(sourceElements2, targetElements2, List(t2)) + val recipe2 = Recipe.createAllToAllRecipe("RECIPE-02", List(m2)) + + val cookbook = Cookbook("COOKBOOK-01", sources, target, List(recipe1, recipe2)) + + val expectedJson = + """ + |{ + | "id" : "COOKBOOK-01", + | "sources" : [ + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-01" + | }, + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-02" + | } + | ], + | "target" : { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "TARGET-ID" + | }, + | "recipes" : [ + | { + | "id" : "RECIPE-01", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "mappingKey" : null, + | "sources" : [ + | {"elementId": "source01", "dfasdlId": "DFASDL"}, + | {"elementId": "source02", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations" : [ + | + | ] + | } + | ] + | }, + | { + | "id" : "RECIPE-02", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "mappingKey" : null, + | "sources" : [ + | {"elementId": "source04", "dfasdlId": "DFASDL"}, + | {"elementId": "source05", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.bar", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "2.71f" + | ], + | [ + | "three", + | "3.00" + | ] + | ] + | } + | } + | ], + | "atomicTransformations" : [ + | + | ] + | } + | ] + | } + | ] + |} + """.stripMargin + + cookbook.asJson.nospaces must be(Parse.parseOption(expectedJson).get.nospaces) + } + } + + describe("decode") { + it("must properly decode json to an object") { + val json = + """ + |{ + | "id" : "COOKBOOK-02", + | "sources" : [ + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-01" + | }, + | { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "SOURCE-02" + | } + | ], + | "target" : { + | "version" : "1.0-SNAPSHOT", + | "content" : "", + | "id" : "TARGET-ID" + | }, + | "recipes" : [ + | { + | "id" : "RECIPE-01", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "sources" : [ + | {"elementId": "source02", "dfasdlId": "DFASDL"}, + | {"elementId": "source03", "dfasdlId": "DFASDL"} + | ], + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations": [] + | } + | ] + | }, + | { + | "id" : "RECIPE-02", + | "mode" : "MapAllToAll", + | "mappings" : [ + | { + | "sources" : [ + | {"elementId": "source04", "dfasdlId": "DFASDL"}, + | {"elementId": "source05", "dfasdlId": "DFASDL"} + | ], + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL"}, + | {"elementId": "target02", "dfasdlId": "DFASDL"}, + | {"elementId": "target03", "dfasdlId": "DFASDL"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.bar", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "2.71f" + | ], + | [ + | "three", + | "3.00" + | ] + | ] + | } + | } + | ], + | "atomicTransformations": [] + | } + | ] + | } + | ] + |} + """.stripMargin + + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val params1 = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o1 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params1) + val t1 = new TransformationDescription("com.example.transformers.foo", o1) + val sourceElements = List( + ElementReference("DFASDL", "source02"), + ElementReference("DFASDL", "source03") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m1 = new MappingTransformation(sourceElements, targetElements, List(t1)) + val recipe1 = Recipe.createAllToAllRecipe("RECIPE-01", List(m1)) + + val params2 = List(("one", "1"), ("two", "2.71f"), ("three", "3.00")) + val o2 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params2) + val t2 = new TransformationDescription("com.example.transformers.bar", o2) + val sourceElements2 = List( + ElementReference("DFASDL", "source04"), + ElementReference("DFASDL", "source05") + ) + val targetElements2 = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m2 = new MappingTransformation(sourceElements2, targetElements2, List(t2)) + val recipe2 = Recipe.createAllToAllRecipe("RECIPE-02", List(m2)) + + val expectedCookbook = Cookbook("COOKBOOK-02", sources, target, List(recipe1, recipe2)) + + Parse.decodeEither[Cookbook](json) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + success must be(expectedCookbook) + } + } + } + } + + describe("findDFASDL") { + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL("TARGET-ID", + "") + ) + + val params1 = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o1 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params1) + val t1 = new TransformationDescription("com.example.transformers.foo", o1) + val sourceElements = List( + ElementReference("DFASDL", "source02"), + ElementReference("DFASDL", "source03") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m1 = new MappingTransformation(sourceElements, targetElements, List(t1)) + val recipe1 = Recipe.createAllToAllRecipe("RECIPE-01", List(m1)) + + val params2 = List(("one", "1"), ("two", "2.71f"), ("three", "3.00")) + val o2 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params2) + val t2 = new TransformationDescription("com.example.transformers.bar", o2) + val sourceElements2 = List( + ElementReference("DFASDL", "source04"), + ElementReference("DFASDL", "source05") + ) + val targetElements2 = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m2 = new MappingTransformation(sourceElements2, targetElements2, List(t2)) + val recipe2 = Recipe.createAllToAllRecipe("RECIPE-02", List(m2)) + + val cookbook = Cookbook("MY-COOKBOOK", sources, target, List(recipe1, recipe2)) + + describe("when using wrong cookbook id") { + it("must return None") { + val ref = DFASDLReference("ANOTHER-COOKBOOK", "SOURCE-01") + + cookbook.findDFASDL(ref) must be(None) + } + } + + describe("when using unknown dfasdl id") { + it("must return None") { + val ref = DFASDLReference("MY-COOKBOOK", "UNKNOWN-DFASDL") + + cookbook.findDFASDL(ref) must be(None) + } + } + + describe("when using the target dfasdl id") { + it("must return an option to the target dfasdl") { + val ref = DFASDLReference("MY-COOKBOOK", "TARGET-ID") + + val dfasdl = cookbook.findDFASDL(ref) + dfasdl.isDefined must be(true) + dfasdl must be(target) + } + } + + describe("when using a source dfasdl id") { + it("must return an option to the source dfasdl") { + val ref = DFASDLReference("MY-COOKBOOK", "SOURCE-02") + + val dfasdl = cookbook.findDFASDL(ref) + dfasdl.isDefined must be(true) + dfasdl.get must be(sources(1)) + } + } + } + + describe("usedSourceIds") { + describe("without mappings") { + it("must return an empty list") { + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val cookbook = Cookbook("MY-COOKBOOK", sources, target, List.empty[Recipe]) + + cookbook.usedSourceIds must be(Set.empty[String]) + } + } + + describe("with mappings") { + it("must return the list of mapped source ids") { + val sources = List( + DFASDL( + "SOURCE-01", + "" + ), + DFASDL("SOURCE-02", "") + ) + val target = Some( + DFASDL( + "TARGET-ID", + "" + ) + ) + + val params1 = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o1 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params1) + val t1 = new TransformationDescription("com.example.transformers.foo", o1) + val sourceElements = List( + ElementReference("DFASDL", "source02"), + ElementReference("DFASDL", "source03") + ) + val targetElements = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m1 = new MappingTransformation(sourceElements, targetElements, List(t1)) + val recipe1 = Recipe.createAllToAllRecipe("RECIPE-01", List(m1)) + + val params2 = List(("one", "1"), ("two", "2.71f"), ("three", "3.00")) + val o2 = new TransformerOptions(classOf[String], classOf[java.lang.Long], params2) + val t2 = new TransformationDescription("com.example.transformers.bar", o2) + val sourceElements2 = List( + ElementReference("DFASDL", "source04"), + ElementReference("DFASDL", "source05"), + ElementReference("DFASDL", "source02") + ) + val targetElements2 = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m2 = new MappingTransformation(sourceElements2, targetElements2, List(t2)) + val recipe2 = Recipe.createAllToAllRecipe("RECIPE-02", List(m2)) + + val cookbook = Cookbook("MY-COOKBOOK", sources, target, List(recipe1, recipe2)) + + val expectedSourceIds = (sourceElements ::: sourceElements2).toSet + + cookbook.usedSourceIds must be(expectedSourceIds) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/DFASDLReferenceTest.scala b/src/test/scala/com/wegtam/tensei/adt/DFASDLReferenceTest.scala new file mode 100644 index 0000000..98c58ed --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/DFASDLReferenceTest.scala @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class DFASDLReferenceTest extends DefaultSpec { + describe("Constraints") { + describe("cookbookId") { + it("must not be empty") { + an[IllegalArgumentException] must be thrownBy DFASDLReference("", "ID") + } + + it("must not be null") { + an[IllegalArgumentException] must be thrownBy DFASDLReference(null, "ID") + } + } + + describe("dfasdlId") { + it("must not be empty") { + an[IllegalArgumentException] must be thrownBy DFASDLReference("ID", "") + } + + it("must not be null") { + an[IllegalArgumentException] must be thrownBy DFASDLReference("ID", null) + } + } + } + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val r = DFASDLReference("COOKBOOK-ID", "DFASDL-ID") + + val expectedJson = """{"cookbook-id":"COOKBOOK-ID","dfasdl-id":"DFASDL-ID"}""" + + r.asJson.nospaces must be(expectedJson) + } + } + + describe("decode") { + it("must properly decode json to an object") { + val json = """{"cookbook-id":"MY-COOKBOOK","dfasdl-id":"MY-DFASDL"}""" + val expectedReference = DFASDLReference("MY-COOKBOOK", "MY-DFASDL") + + val decoded = Parse.decodeOption[DFASDLReference](json) + decoded.isDefined must be(true) + decoded.get must be(expectedReference) + decoded.get.cookbookId must be(expectedReference.cookbookId) + decoded.get.dfasdlId must be(expectedReference.dfasdlId) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/DFASDLTest.scala b/src/test/scala/com/wegtam/tensei/adt/DFASDLTest.scala new file mode 100644 index 0000000..79dbac0 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/DFASDLTest.scala @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class DFASDLTest extends DefaultSpec { + describe("DFASDL") { + describe("CodecJson") { + describe("encode") { + it("must properly encode an object to json") { + val dfasdl = DFASDL( + "ID-01", + "", + "0.5.0" + ) + + val expectedJson = + """ + |{ + | "version" : "0.5.0", + | "content" : "", + | "id" : "ID-01" + |} + """.stripMargin + + dfasdl.asJson.nospaces must be(Parse.parseOption(expectedJson).get.nospaces) + } + } + + describe("decode") { + it("must properly decode json to an object") { + val json = + """ + |{ + | "version" : "2.0", + | "content" : "", + | "id" : "ID-01" + |} + """.stripMargin + + val expectedDfasdl = DFASDL( + "ID-01", + "", + "2.0" + ) + + val actualDfasdl = Parse.decodeOption[DFASDL](json) + actualDfasdl.isDefined must be(true) + actualDfasdl.get must be(expectedDfasdl) + actualDfasdl.get.id must be(expectedDfasdl.id) + actualDfasdl.get.content must be(expectedDfasdl.content) + actualDfasdl.get.version must be(expectedDfasdl.version) + } + } + } + + describe("autogenerateMissingIds") { + describe("on an empty dfasdl") { + it("must return the original dfasdl") { + val dfasdl = DFASDL("ID-01", "", "1.0") + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(dfasdl) + returnedDfasdl.content must equal(dfasdl.content) + } + } + + describe("on a dfasdl without missing ids") { + it("must return the original dfasdl") { + val dfasdl = DFASDL( + "ID-01", + "", + "1.0" + ) + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(dfasdl) + returnedDfasdl.content must equal(dfasdl.content) + } + } + + describe("on a dfasdl with missing ids") { + describe("without auto-generated ids") { + it("must auto-generate the missing ids") { + val dfasdl = DFASDL( + "ID-01", + "", + "1.0" + ) + val expectedDfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(expectedDfasdl) + returnedDfasdl.content must equal(expectedDfasdl.content) + } + } + + describe("without auto-generated ids and with empty ids") { + it("must auto-generate the missing and empty ids") { + val dfasdl = DFASDL( + "ID-01", + "", + "1.0" + ) + val expectedDfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(expectedDfasdl) + returnedDfasdl.content must equal(expectedDfasdl.content) + } + } + + describe("with auto-generated ids") { + it("must auto-generate the missing ids using the last proper auto-generated value") { + val dfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val expectedDfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(expectedDfasdl) + returnedDfasdl.content must equal(expectedDfasdl.content) + } + } + + describe("with auto-generated ids and with empty ids") { + it("must auto-generate the missing ids using the last proper auto-generated value") { + val dfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val expectedDfasdl = DFASDL( + "ID-01", + s"""""", + "1.0" + ) + val returnedDfasdl = DFASDL.autogenerateMissingIds(dfasdl) + returnedDfasdl must equal(expectedDfasdl) + returnedDfasdl.content must equal(expectedDfasdl.content) + } + } + } + } + + describe("equals") { + describe("if compared with itself") { + it("must return true") { + val dfasdl = new DFASDL(id = "FOO", content = "I am not a valid DFASDL!") + + dfasdl.equals(dfasdl) must be(true) + (dfasdl == dfasdl) must be(true) + } + } + + describe("if id, content and version are equal") { + it("must return true") { + val one = new DFASDL(id = "FOO", content = "I am not a valid DFASDL!") + val two = new DFASDL(id = "FOO", content = "I am not a valid DFASDL!") + + one.equals(two) must be(true) + (one == two) must be(true) + } + } + + describe("if id, content or version differ") { + it("must return false") { + val dfasdl = + new DFASDL(id = "FOO", content = "I am not a valid DFASDL!", version = "1.0") + val otherId = dfasdl.copy(id = "BAR") + val otherContent = dfasdl.copy(content = "I am also not a valid DFASDL! :-)") + val otherVersion = dfasdl.copy(version = "0.1-alpha") + + dfasdl.equals(otherId) must be(false) + (dfasdl == otherId) must be(false) + + dfasdl.equals(otherContent) must be(false) + (dfasdl == otherContent) must be(false) + + dfasdl.equals(otherVersion) must be(false) + (dfasdl == otherVersion) must be(false) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/ElementReferenceTest.scala b/src/test/scala/com/wegtam/tensei/adt/ElementReferenceTest.scala new file mode 100644 index 0000000..9d45697 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/ElementReferenceTest.scala @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +import scalaz._ + +class ElementReferenceTest extends DefaultSpec { + describe("ElementReference") { + describe("Constraints") { + describe("dfasdlId") { + it("must not be empty") { + an[IllegalArgumentException] must be thrownBy ElementReference(dfasdlId = "", + elementId = "ID") + } + + it("must not be null") { + an[IllegalArgumentException] must be thrownBy ElementReference(dfasdlId = null, + elementId = "ID") + } + } + + describe("elementId") { + it("must not be empty") { + an[IllegalArgumentException] must be thrownBy ElementReference(dfasdlId = "ID", + elementId = "") + } + + it("must not be null") { + an[IllegalArgumentException] must be thrownBy ElementReference(dfasdlId = "ID", + elementId = null) + } + } + } + + describe("JsonCodec") { + describe("decode") { + it("must decode a proper json string correctly") { + val expectedRef = + ElementReference(dfasdlId = "MY-DFASDL-ID", elementId = "MY-ELEMENT-ID") + val json = + s""" + |{ + | "dfasdlId": "${expectedRef.dfasdlId}", + | "elementId": "${expectedRef.elementId}" + |} + """.stripMargin + + Parse.decodeEither[ElementReference](json) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + success must be(expectedRef) + success.dfasdlId must be(expectedRef.dfasdlId) + success.elementId must be(expectedRef.elementId) + } + } + } + + describe("encode") { + it("must produce a correct json string") { + val ref = ElementReference(dfasdlId = "MY-DFASDL-ID", elementId = "MY-ELEMENT-ID") + val expectedJson = + s""" + |{ + | "elementId": "${ref.elementId}", + | "dfasdlId": "${ref.dfasdlId}" + |} + """.stripMargin + + val encodedRef = ref.asJson + Parse.parse(expectedJson) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + encodedRef must be(success) + encodedRef.nospaces must be(success.nospaces) // TODO We check the string here because we're unsure if the test above is enough. + } + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/ExtractSchemaOptionsTest.scala b/src/test/scala/com/wegtam/tensei/adt/ExtractSchemaOptionsTest.scala new file mode 100644 index 0000000..4fa1ff8 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/ExtractSchemaOptionsTest.scala @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import java.nio.charset.Charset + +import com.wegtam.tensei.DefaultSpec +import org.scalacheck.Gen +import org.scalatest.prop.PropertyChecks + +import scala.collection.JavaConverters._ + +class ExtractSchemaOptionsTest extends DefaultSpec with PropertyChecks { + + private val charsets = Charset.availableCharsets().keySet().asScala.toVector :+ "" + private val csvOptions = for { + header ← Gen.oneOf(false, true) + separator ← Gen.alphaNumStr + encoding ← Gen.oneOf(charsets) + } yield (header, separator, encoding) + + describe("#createCsvOptions") { + it("must create proper options") { + forAll(csvOptions) { o ⇒ + val (h, s, e) = o + val expectedOptions = ExtractSchemaOptions( + csvHeader = h, + csvSeparator = if (s.isEmpty) None else Option(s), + encoding = if (e.isEmpty) None else Option(e) + ) + ExtractSchemaOptions.createCsvOptions(h, s, e) mustEqual expectedOptions + } + } + } + + describe("#createDatabaseOptions") { + it("must create proper options") { + val expectedOptions = + ExtractSchemaOptions(csvHeader = false, csvSeparator = None, encoding = None) + ExtractSchemaOptions.createDatabaseOptions() mustEqual expectedOptions + } + } + +} diff --git a/src/test/scala/com/wegtam/tensei/adt/MappingTransformationTest.scala b/src/test/scala/com/wegtam/tensei/adt/MappingTransformationTest.scala new file mode 100644 index 0000000..b78dff1 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/MappingTransformationTest.scala @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.DefaultSpec + +import scalaz._ + +class MappingTransformationTest extends DefaultSpec { + describe("when no target id is specified") { + it("must be invalid") { + an[IllegalArgumentException] must be thrownBy MappingTransformation( + List(ElementReference("dfasdl", "source")), + List() + ) + } + } + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val at = new AtomicTransformationDescription(ElementReference("DFASDL-1", "source01"), + "com.example.transformers.bar", + TransformerOptions(classOf[String], + classOf[String], + List())) + val sources = List( + ElementReference("DFASDL-1", "source01"), + ElementReference("DFASDL-1", "source02") + ) + val targets = List( + ElementReference("DFASDL-2", "target01"), + ElementReference("DFASDL-2", "target02"), + ElementReference("DFASDL-2", "target03") + ) + val m = new MappingTransformation(sources, targets, List(t), List(at)) + val expectedJson = + """ + |{ + | "targets" : [ + | {"elementId": "target01", "dfasdlId": "DFASDL-2"}, + | {"elementId": "target02", "dfasdlId": "DFASDL-2"}, + | {"elementId": "target03", "dfasdlId": "DFASDL-2"} + | ], + | "mappingKey" : null, + | "sources" : [ + | {"elementId": "source01", "dfasdlId": "DFASDL-1"}, + | {"elementId": "source02", "dfasdlId": "DFASDL-1"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "1" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations" : [ + | { + | "element" : { + | "elementId": "source01", + | "dfasdlId": "DFASDL-1" + | }, + | "transformerClassName" : "com.example.transformers.bar", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.String", + | "params" : [ + | + | ] + | } + | } + | ] + |} + """.stripMargin + + m.asJson.nospaces mustEqual Parse.parseOption(expectedJson).get.nospaces + } + } + + describe("decode") { + it("must properly decode json to an object") { + val jsonString = + """ + |{ + | "sources" : [ + | {"elementId": "sourceABC", "dfasdlId": "DFASDL-1"}, + | {"elementId": "source02", "dfasdlId": "DFASDL-1"} + | ], + | "targets" : [ + | {"elementId": "target-XYZ", "dfasdlId": "DFASDL-2"}, + | {"elementId": "target03", "dfasdlId": "DFASDL-2"} + | ], + | "transformations" : [ + | { + | "transformerClassName" : "com.example.transformers.foo", + | "options" : { + | "srcType" : "java.lang.Integer", + | "dstType" : "java.lang.Long", + | "params" : [ + | [ + | "one", + | "ABC" + | ], + | [ + | "two", + | "Zwei" + | ], + | [ + | "three", + | "3.14f" + | ] + | ] + | } + | } + | ], + | "atomicTransformations" : [ + | { + | "element" : { + | "dfasdlId": "DFASDL-1", + | "elementId": "sourceABC" + | }, + | "transformerClassName" : "com.example.transformers.bar", + | "options" : { + | "srcType" : "java.lang.String", + | "dstType" : "java.lang.String", + | "params" : [ + | + | ] + | } + | } + | ] + |} + """.stripMargin + val params = List(("one", "ABC"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[Integer], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val at = new AtomicTransformationDescription(ElementReference("DFASDL-1", "sourceABC"), + "com.example.transformers.bar", + TransformerOptions(classOf[String], + classOf[String], + List())) + val sources = List( + ElementReference("DFASDL-1", "sourceABC"), + ElementReference("DFASDL-1", "source02") + ) + val targets = List( + ElementReference("DFASDL-2", "target-XYZ"), + ElementReference("DFASDL-2", "target03") + ) + val expected = new MappingTransformation(sources, targets, List(t), List(at)) + Parse.decodeEither[MappingTransformation](jsonString) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ success must be(expected) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/RecipeTest.scala b/src/test/scala/com/wegtam/tensei/adt/RecipeTest.scala new file mode 100644 index 0000000..e9b845e --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/RecipeTest.scala @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.DefaultSpec +import com.wegtam.tensei.adt.Recipe.MapAllToAll + +import scalaz._ + +class RecipeTest extends DefaultSpec { + describe("when created without an id") { + it("must be invalid") { + an[IllegalArgumentException] must be thrownBy new Recipe( + "", + MapAllToAll, + List( + new MappingTransformation(List(ElementReference("DFASDL", "source")), + List(ElementReference("DFASDL", "target"))) + ) + ) + } + } + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("http://www.example.com", o) + val sources = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targets = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sources, targets, List(t)) + val r = Recipe.createAllToAllRecipe("my-recipe", List(m)) + val expectedJson = + """{"id":"my-recipe","mode":"MapAllToAll","mappings":[{"targets":[{"elementId":"target01","dfasdlId":"DFASDL"},{"elementId":"target02","dfasdlId":"DFASDL"},{"elementId":"target03","dfasdlId":"DFASDL"}],"transformations":[{"transformerClassName":"http://www.example.com","options":{"srcType":"java.lang.String","dstType":"java.lang.Long","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}}],"atomicTransformations":[],"mappingKey":null,"sources":[{"elementId":"source01","dfasdlId":"DFASDL"},{"elementId":"source02","dfasdlId":"DFASDL"}]}]}""" + r.asJson.nospaces mustEqual expectedJson + } + } + + describe("decode") { + it("must properly decode json to an object") { + val jsonString = + """{"id":"my-recipe","mode":"MapAllToAll","mappings":[{"sources":[{"elementId":"source01","dfasdlId":"DFASDL"},{"elementId":"source02","dfasdlId":"DFASDL"}],"targets":[{"elementId":"target01","dfasdlId":"DFASDL"},{"elementId":"target02","dfasdlId":"DFASDL"},{"elementId":"target03","dfasdlId":"DFASDL"}],"transformations":[{"transformerClassName":"http://www.example.com","options":{"srcType":"java.lang.String","dstType":"java.lang.Long","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}}],"atomicTransformations":[]}]}""" + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("http://www.example.com", o) + val sources = List( + ElementReference("DFASDL", "source01"), + ElementReference("DFASDL", "source02") + ) + val targets = List( + ElementReference("DFASDL", "target01"), + ElementReference("DFASDL", "target02"), + ElementReference("DFASDL", "target03") + ) + val m = new MappingTransformation(sources, targets, List(t)) + val expected = Recipe.createAllToAllRecipe("my-recipe", List(m)) + Parse.decodeEither[Recipe](jsonString) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ success must be(expected) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/RuntimeStats$Test.scala b/src/test/scala/com/wegtam/tensei/adt/RuntimeStats$Test.scala new file mode 100644 index 0000000..73c7b2e --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/RuntimeStats$Test.scala @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class RuntimeStats$Test extends DefaultSpec { + describe("RuntimeStats") { + describe("CodecJson") { + describe("decode") { + it("must decode a proper json string") { + val expected = + RuntimeStats(freeMemory = 61096024, maxMemory = 239075328, totalMemory = 102760448) + Parse + .decodeOption[RuntimeStats]( + """{"processors":1,"load":null,"total": 102760448, "max": 239075328, "free": 61096024}""" + ) + .get must be(expected) + val expectedOSData = RuntimeStats(freeMemory = 61096024, + maxMemory = 239075328, + totalMemory = 102760448, + processors = 4, + systemLoad = Option(2.6d)) + Parse + .decodeOption[RuntimeStats]( + """{"processors":4,"load":2.6,"total": 102760448, "max": 239075328, "free": 61096024}""" + ) + .get must be(expectedOSData) + } + } + + describe("encode") { + it("must encode to a proper json string") { + val stats = + RuntimeStats(freeMemory = 61096024, maxMemory = 239075328, totalMemory = 102760448) + stats.asJson.nospaces must be( + """{"processors":1,"load":null,"total":102760448,"max":239075328,"free":61096024}""" + ) + val statsWithOSData = RuntimeStats(freeMemory = 61096024, + maxMemory = 239075328, + totalMemory = 102760448, + processors = 12, + systemLoad = Option(8.9d)) + statsWithOSData.asJson.nospaces must be( + """{"processors":12,"load":8.9,"total":102760448,"max":239075328,"free":61096024}""" + ) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/StatsMessagesTest.scala b/src/test/scala/com/wegtam/tensei/adt/StatsMessagesTest.scala new file mode 100644 index 0000000..25d0e94 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/StatsMessagesTest.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import java.net.URI + +import com.wegtam.tensei.DefaultSpec +import com.wegtam.tensei.adt.StatsMessages.CalculateStatisticsResult +import com.wegtam.tensei.adt.StatsResult.{ + BasicStatisticsResult, + StatsResultNumeric, + StatsResultString +} + +import scalaz._, Scalaz._ + +class StatsMessagesTest extends DefaultSpec { + describe("StatsMessages") { + describe("CalculateStatisticsResult") { + val dfasdl = DFASDL("SIMPLE-DFASDL", "") + val refs = List( + ElementReference("DFASDL", "alter"), + ElementReference("DFASDL", "name") + ) + val mapping = MappingTransformation(refs, refs) + val recipe = new Recipe("COPY-COLUMNS", Recipe.MapOneToOne, List(mapping)) + val cookbook = Cookbook("COOKBOOK", List(dfasdl), Option(dfasdl), List(recipe)) + val source = + ConnectionInformation(new URI(""), Option(DFASDLReference(cookbook.id, dfasdl.id))) + + describe("with numerical and string results") { + it("must work") { + val numericResult = new StatsResultNumeric( + "alter", + new BasicStatisticsResult(3, Option(3), Option(1.0), Option(5.0), Option(3.0)) + ) + val stringResult = new StatsResultString( + "name", + new BasicStatisticsResult(3, Option(3), Option(1.0), Option(5.0), Option(3.0)) + ) + new CalculateStatisticsResult(List(numericResult, stringResult).right[String], + source = source, + cookbook, + List("alter", "name")) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/StatusMessageTest.scala b/src/test/scala/com/wegtam/tensei/adt/StatusMessageTest.scala new file mode 100644 index 0000000..c1316fc --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/StatusMessageTest.scala @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class StatusMessageTest extends DefaultSpec { + class TestStatusMessage(reporter: Option[String] = None, + message: String, + statusType: StatusType = StatusType.MinorError, + cause: Option[StatusMessage] = None) + extends StatusMessage(reporter, message, statusType, cause) + + describe("StatusMessage") { + describe("EncodeJson") { + describe("using simple status messages") { + it("must encode proper json") { + val e = new TestStatusMessage(message = "I am a status message!") + e.asJson.nospaces must be( + """{"reporter":null,"message":"I am a status message!","statusType":"MinorError","cause":null}""" + ) + } + } + + describe("using chained status messages") { + it("must encode proper json") { + val foo = new TestStatusMessage(message = "I am an status message!", + statusType = StatusType.MajorError) + val bar = new TestStatusMessage(message = "I am caused by the previous status!", + statusType = StatusType.FatalError, + cause = Option(foo)) + bar.asJson.nospaces must be( + """{"reporter":null,"message":"I am caused by the previous status!","statusType":"FatalError","cause":{"reporter":null,"message":"I am an status message!","statusType":"MajorError","cause":null}}""" + ) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/StatusTypeTest.scala b/src/test/scala/com/wegtam/tensei/adt/StatusTypeTest.scala new file mode 100644 index 0000000..7d7ba6f --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/StatusTypeTest.scala @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class StatusTypeTest extends DefaultSpec { + describe("StatusType") { + describe("CodecJson") { + describe("decode") { + it("must decode all known types correctly") { + Parse.decodeOption[StatusType](""""MinorError"""").get must be(StatusType.MinorError) + Parse.decodeOption[StatusType](""""MajorError"""").get must be(StatusType.MajorError) + Parse.decodeOption[StatusType](""""NoAgentAvailable"""").get must be( + StatusType.NoAgentAvailable + ) + Parse.decodeOption[StatusType](""""FatalError"""").get must be(StatusType.FatalError) + } + + it("must throw an IllegalArgumentException upon an unknown type") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[StatusType](""""SomeUnknownType"""") + ) + } + } + + describe("encode") { + it("must encode all known types correctly") { + StatusType.MinorError.asJson.nospaces must be(""""MinorError"""") + StatusType.MajorError.asJson.nospaces must be(""""MajorError"""") + StatusType.NoAgentAvailable.asJson.nospaces must be(""""NoAgentAvailable"""") + StatusType.FatalError.asJson.nospaces must be(""""FatalError"""") + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/TenseiLicenseTest.scala b/src/test/scala/com/wegtam/tensei/adt/TenseiLicenseTest.scala new file mode 100644 index 0000000..a4e20dd --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/TenseiLicenseTest.scala @@ -0,0 +1,355 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ + +import java.time.LocalDate + +import scalaz._ + +import com.wegtam.tensei.DefaultSpec + +class TenseiLicenseTest extends DefaultSpec { + describe("TenseiLicense") { + describe("createEnterpriseLicense") { + describe("given valid parameters") { + it("must create an enterprise license") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now().plusYears(1L) + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 5, + users = Int.MaxValue, + configurations = Int.MaxValue, + cronjobs = Int.MaxValue, + triggers = Int.MaxValue, + expirationDate = expirationDate + ) + + val actualLicense = TenseiLicense.createEnterpriseLicense(licensee, expirationDate) + + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + + describe("given the expiration date of today") { + it("must create an enterprise license") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now() + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 5, + users = Int.MaxValue, + configurations = Int.MaxValue, + cronjobs = Int.MaxValue, + triggers = Int.MaxValue, + expirationDate = expirationDate + ) + + val actualLicense = TenseiLicense.createEnterpriseLicense(licensee, expirationDate) + + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + + describe("given an expiration date from the past") { + it("must throw an IllegalArgumentException") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now().minusDays(1L) + + a[IllegalArgumentException] must be thrownBy TenseiLicense.createEnterpriseLicense( + licensee, + expirationDate + ) + } + } + } + + describe("createProfessionalLicense") { + describe("given valid parameters") { + it("must create a professional license") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now().plusYears(1L) + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = expirationDate + ) + + val actualLicense = TenseiLicense.createProfessionalLicense(licensee, expirationDate) + + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + + describe("given the expiration date of today") { + it("must create a professional license") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now() + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = expirationDate + ) + + val actualLicense = TenseiLicense.createProfessionalLicense(licensee, expirationDate) + + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + + describe("given an expiration date from the past") { + it("must throw an IllegalArgumentException") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now().minusDays(1L) + + a[IllegalArgumentException] must be thrownBy TenseiLicense.createProfessionalLicense( + licensee, + expirationDate + ) + } + } + } + + describe("createTestLicense") { + it("must create a license that expires within a month") { + val licensee = "TEST-LICENSEE" + val expirationDate = LocalDate.now().plusMonths(1L) + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = expirationDate + ) + + val actualLicense = TenseiLicense.createTestLicense(licensee) + + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + + describe("expiresIn") { + describe("when license is not expired") { + it("must return the period until the license expires") { + val licensee = "TEST-LICENSEE" + + val oneYearLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now().plusYears(1L) + ) + oneYearLicense.expiresIn.getYears must be(1) + + val oneMonthLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now().plusMonths(1L) + ) + oneMonthLicense.expiresIn.getMonths must be(1) + + val oneDayLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now().plusDays(1L) + ) + oneDayLicense.expiresIn.getDays must be(1) + } + } + + describe("when the license expires today") { + it("must return a period that is zero") { + val licensee = "TEST-LICENSEE" + + val license = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now() + ) + license.expiresIn.isZero must be(right = true) + } + } + + describe("when the license has expired") { + it("must return a negative period") { + val licensee = "TEST-LICENSEE" + + val license = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now().minusDays(1L) + ) + license.expiresIn.isNegative must be(right = true) + } + } + } + + describe("CodecJson") { + describe("decode") { + it("must decode proper json") { + val licensee = "TEST-LICENSEE" + + val expectedLicense = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = scala.util.Random.nextInt(), + users = scala.util.Random.nextInt(), + configurations = scala.util.Random.nextInt(), + cronjobs = scala.util.Random.nextInt(), + triggers = scala.util.Random.nextInt(), + expirationDate = LocalDate.now().plusYears(1L).plusMonths(6L) + ) + val jsonString = + s""" + |{ + | "cronjobs" : ${expectedLicense.cronjobs}, + | "licensee" : "$licensee", + | "users" : ${expectedLicense.users}, + | "agents" : ${expectedLicense.agents}, + | "expiration-date" : "${expectedLicense.expirationDate.toString}", + | "triggers" : ${expectedLicense.triggers}, + | "id" : "${expectedLicense.id}", + | "configurations" : ${expectedLicense.configurations} + |} + """.stripMargin.trim + + Parse.decodeEither[TenseiLicense](jsonString) match { + case -\/(error) ⇒ fail(error) + case \/-(actualLicense) ⇒ + actualLicense.licensee must be(expectedLicense.licensee) + actualLicense.agents must be(expectedLicense.agents) + actualLicense.users must be(expectedLicense.users) + actualLicense.configurations must be(expectedLicense.configurations) + actualLicense.cronjobs must be(expectedLicense.cronjobs) + actualLicense.triggers must be(expectedLicense.triggers) + actualLicense.expirationDate must be(expectedLicense.expirationDate) + } + } + } + + describe("encode") { + it("must encode to proper json") { + val licensee = "TEST-LICENSEE" + + val license = TenseiLicense( + id = java.util.UUID.randomUUID().toString, + licensee = licensee, + agents = 1, + users = 1, + configurations = 3, + cronjobs = 3, + triggers = 3, + expirationDate = LocalDate.now().plusMonths(6L) + ) + val expectedParts = List( + s""""cronjobs" : ${license.cronjobs},""", + s""""licensee" : "$licensee",""", + s""""users" : ${license.users},""", + s""""agents" : ${license.agents},""", + s""""expiration-date" : "${license.expirationDate.toString}",""", + s""""triggers" : ${license.triggers},""", + s""""id" : "${license.id}",""", + s""""configurations" : ${license.configurations}""", + s""""playload" : """" + ) + + val actualJson = license.asJson.spaces2 + expectedParts.foreach { part ⇒ + actualJson must include(part) + } + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/TransformationDataType$Test.scala b/src/test/scala/com/wegtam/tensei/adt/TransformationDataType$Test.scala new file mode 100644 index 0000000..45b90b7 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/TransformationDataType$Test.scala @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import com.wegtam.tensei.DefaultSpec + +class TransformationDataType$Test extends DefaultSpec { + describe("isValidDataType") { + describe("the following data types must be valid") { + it("String") { + TransformationDataType.isValidDataType(classOf[String]) must be(true) + } + + it("Integer") { + TransformationDataType.isValidDataType(classOf[Integer]) must be(true) + } + + it("Long") { + TransformationDataType.isValidDataType(classOf[java.lang.Long]) must be(true) + } + + it("Short") { + TransformationDataType.isValidDataType(classOf[java.lang.Short]) must be(true) + } + + it("Byte") { + TransformationDataType.isValidDataType(classOf[java.lang.Byte]) must be(true) + } + + it("Character") { + TransformationDataType.isValidDataType(classOf[Character]) must be(true) + } + + it("Float") { + TransformationDataType.isValidDataType(classOf[java.lang.Float]) must be(true) + } + + it("Double") { + TransformationDataType.isValidDataType(classOf[java.lang.Double]) must be(true) + } + + it("Boolean") { + TransformationDataType.isValidDataType(classOf[java.lang.Boolean]) must be(true) + } + + it("Array[Byte]") { + TransformationDataType.isValidDataType(classOf[Array[Byte]]) must be(true) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/TransformationDescriptionTest.scala b/src/test/scala/com/wegtam/tensei/adt/TransformationDescriptionTest.scala new file mode 100644 index 0000000..c1c9bc7 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/TransformationDescriptionTest.scala @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.DefaultSpec + +class TransformationDescriptionTest extends DefaultSpec { + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[String], classOf[java.lang.Long], params) + val t = new TransformationDescription("com.example.transformers.foo", o) + val expectedJson = + """{"transformerClassName":"com.example.transformers.foo","options":{"srcType":"java.lang.String","dstType":"java.lang.Long","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}}""" + t.asJson.nospaces mustEqual expectedJson + } + } + + describe("decode") { + it("must properly decode json to an object") { + val params = List(("one", "1"), ("two", "Zweikommasechs"), ("three", "3.14f")) + val o = new TransformerOptions(classOf[java.lang.Integer], classOf[java.lang.Long], params) + val expected = new TransformationDescription("com.example.transformers.foo", o) + val jsonString = + """{"transformerClassName":"com.example.transformers.foo","options":{"srcType":"java.lang.Integer","dstType":"java.lang.Long","params":[["one","1"],["two","Zweikommasechs"],["three","3.14f"]]}}""" + val decoded: Option[TransformationDescription] = + Parse.decodeOption[TransformationDescription](jsonString) + decoded.isDefined must be(true) + decoded.get must be(expected) + decoded.get.transformerClassName must be(expected.transformerClassName) + decoded.get.options must be(expected.options) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/adt/TransformerOptionsTest.scala b/src/test/scala/com/wegtam/tensei/adt/TransformerOptionsTest.scala new file mode 100644 index 0000000..db5a8cb --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/adt/TransformerOptionsTest.scala @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.adt + +import argonaut._, Argonaut._ +import com.wegtam.tensei.DefaultSpec + +class TransformerOptionsTest extends DefaultSpec { + describe("when creating a new transformer options object") { + describe("it must only allow certain value types") { + it("String must be valid") { + val t = new TransformerOptions(classOf[String], classOf[String]) + t.srcType must be(classOf[String]) + t.dstType must be(classOf[String]) + } + + it("Array[Byte] must be valid") { + val t = new TransformerOptions(classOf[Array[Byte]], classOf[Array[Byte]]) + t.srcType must be(classOf[Array[Byte]]) + t.dstType must be(classOf[Array[Byte]]) + } + + it("Boolean and String must be valid") { + val t = new TransformerOptions(classOf[java.lang.Boolean], classOf[String]) + t.srcType must be(classOf[java.lang.Boolean]) + t.dstType must be(classOf[String]) + } + + it("Integer and Long must be valid") { + val t = new TransformerOptions(classOf[Integer], classOf[java.lang.Long]) + t.srcType must be(classOf[Integer]) + t.dstType must be(classOf[java.lang.Long]) + } + + it("Character and Byte must be valid") { + val t = new TransformerOptions(classOf[Character], classOf[java.lang.Byte]) + t.srcType must be(classOf[Character]) + t.dstType must be(classOf[java.lang.Byte]) + } + } + + describe("when given a non empty list of parameters") { + it("must accept a list of Tuple2(String, Any)") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val t = new TransformerOptions(classOf[String], classOf[String], params) + t.params must be(params) + } + } + } + + describe("JsonCodec") { + describe("encode") { + it("must properly encode an object to json") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val t = new TransformerOptions(classOf[String], classOf[String], params) + val expectedJson = + """{"srcType":"java.lang.String","dstType":"java.lang.String","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}""" + t.asJson.nospaces must be(expectedJson) + } + } + + describe("decode") { + it("must properly decode json to an object") { + val params = List(("one", "1"), ("two", "Zwei"), ("three", "3.14f")) + val expected = new TransformerOptions(classOf[String], classOf[String], params) + val jsonString = + """{"srcType":"java.lang.String","dstType":"java.lang.String","params":[["one","1"],["two","Zwei"],["three","3.14f"]]}""" + val decoded: Option[TransformerOptions] = + Parse.decodeOption[TransformerOptions](jsonString) + decoded.isDefined must be(true) + decoded.get must be(expected) + decoded.get.srcType must be(expected.srcType) + decoded.get.dstType must be(expected.dstType) + decoded.get.params must be(expected.params) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/agent/ParserStateTest.scala b/src/test/scala/com/wegtam/tensei/agent/ParserStateTest.scala new file mode 100644 index 0000000..1ee9e75 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/agent/ParserStateTest.scala @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class ParserStateTest extends DefaultSpec { + describe("ParserState") { + describe("CodecJson") { + describe("decode") { + it("must decode all known states correctly") { + Parse.decodeOption[ParserState](""""Idle"""").get must be(ParserState.Idle) + Parse.decodeOption[ParserState](""""ValidatingSyntax"""").get must be( + ParserState.ValidatingSyntax + ) + Parse.decodeOption[ParserState](""""ValidatingAccess"""").get must be( + ParserState.ValidatingAccess + ) + Parse.decodeOption[ParserState](""""ValidatingChecksums"""").get must be( + ParserState.ValidatingChecksums + ) + Parse.decodeOption[ParserState](""""PreparingSourceData"""").get must be( + ParserState.PreparingSourceData + ) + Parse.decodeOption[ParserState](""""InitializingSubParsers"""").get must be( + ParserState.InitializingSubParsers + ) + Parse.decodeOption[ParserState](""""Parsing"""").get must be(ParserState.Parsing) + } + + it("must throw an IllegalArgumentException upon an unknown state") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[ParserState](""""SomeUnknownState"""") + ) + } + } + + describe("encode") { + it("must encode all known states correctly") { + ParserState.Idle.asJson.nospaces must be(""""Idle"""") + ParserState.ValidatingSyntax.asJson.nospaces must be(""""ValidatingSyntax"""") + ParserState.ValidatingAccess.asJson.nospaces must be(""""ValidatingAccess"""") + ParserState.ValidatingChecksums.asJson.nospaces must be(""""ValidatingChecksums"""") + ParserState.PreparingSourceData.asJson.nospaces must be(""""PreparingSourceData"""") + ParserState.InitializingSubParsers.asJson.nospaces must be( + """"InitializingSubParsers"""" + ) + ParserState.Parsing.asJson.nospaces must be(""""Parsing"""") + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/agent/ProcessorStateTest.scala b/src/test/scala/com/wegtam/tensei/agent/ProcessorStateTest.scala new file mode 100644 index 0000000..a251a37 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/agent/ProcessorStateTest.scala @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class ProcessorStateTest extends DefaultSpec { + describe("ProcessorState") { + describe("CodecJson") { + describe("decode") { + it("must decode all known states correctly") { + Parse.decodeOption[ProcessorState](""""Idle"""").get must be(ProcessorState.Idle) + Parse.decodeOption[ProcessorState](""""Sorting"""").get must be(ProcessorState.Sorting) + Parse.decodeOption[ProcessorState](""""Processing"""").get must be( + ProcessorState.Processing + ) + Parse.decodeOption[ProcessorState](""""WaitingForWriterClosing"""").get must be( + ProcessorState.WaitingForWriterClosing + ) + } + + it("must throw an IllegalArgumentException upon an unknown state") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[ProcessorState](""""SomeUnknownState"""") + ) + } + } + + describe("encode") { + it("must encode all known states correctly") { + ProcessorState.Idle.asJson.nospaces must be(""""Idle"""") + ProcessorState.Sorting.asJson.nospaces must be(""""Sorting"""") + ProcessorState.Processing.asJson.nospaces must be(""""Processing"""") + ProcessorState.WaitingForWriterClosing.asJson.nospaces must be( + """"WaitingForWriterClosing"""" + ) + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/agent/TenseiAgentStateTest.scala b/src/test/scala/com/wegtam/tensei/agent/TenseiAgentStateTest.scala new file mode 100644 index 0000000..ed1c654 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/agent/TenseiAgentStateTest.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.agent + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class TenseiAgentStateTest extends DefaultSpec { + describe("TenseiAgentState") { + describe("CodecJson") { + describe("decode") { + it("must decode all known states correctly") { + Parse.decodeOption[TenseiAgentState](""""Aborting"""").get must be( + TenseiAgentState.Aborting + ) + Parse.decodeOption[TenseiAgentState](""""CleaningUp"""").get must be( + TenseiAgentState.CleaningUp + ) + Parse.decodeOption[TenseiAgentState](""""Idle"""").get must be(TenseiAgentState.Idle) + Parse.decodeOption[TenseiAgentState](""""InitializingResources"""").get must be( + TenseiAgentState.InitializingResources + ) + Parse.decodeOption[TenseiAgentState](""""Working"""").get must be( + TenseiAgentState.Working + ) + } + + it("must throw an IllegalArgumentException upon an unknown state") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[TenseiAgentState](""""SomeUnknownState"""") + ) + } + } + + describe("encode") { + it("must encode all known states correctly") { + TenseiAgentState.Aborting.asJson.nospaces must be(""""Aborting"""") + TenseiAgentState.CleaningUp.asJson.nospaces must be(""""CleaningUp"""") + TenseiAgentState.Idle.asJson.nospaces must be(""""Idle"""") + TenseiAgentState.InitializingResources.asJson.nospaces must be( + """"InitializingResources"""" + ) + TenseiAgentState.Working.asJson.nospaces must be(""""Working"""") + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/security/CryptoHelpersTest.scala b/src/test/scala/com/wegtam/tensei/security/CryptoHelpersTest.scala new file mode 100644 index 0000000..5655c37 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/security/CryptoHelpersTest.scala @@ -0,0 +1,308 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.security + +import argonaut._, Argonaut._ + +import java.security.KeyFactory +import java.security.interfaces.{ RSAPrivateKey, RSAPublicKey } +import java.security.spec.{ RSAPrivateKeySpec, RSAPublicKeySpec } +import java.time.LocalDate +import javax.crypto.{ Cipher, IllegalBlockSizeException } + +import com.wegtam.tensei.DefaultSpec +import com.wegtam.tensei.adt.TenseiLicense + +import scalaz._ + +class CryptoHelpersTest extends DefaultSpec with CryptoHelpers { + describe("CryptoHelpers") { + describe("default AES key length") { + it("must be 128") { + DEFAULT_AES_KEY_LENGTH must be(128) + } + } + + describe("default AES variant") { + it("must be AES/CBC/PKCS5PADDING") { + DEFAULT_AES_VARIANT must be("AES/CBC/PKCS5PADDING") + } + } + + describe("default RSA key length") { + it("must be 2048") { + DEFAULT_RSA_KEY_LENGTH must be(2048) + } + } + + describe("default RSA variant") { + it("must be RSA/ECB/PKCS1Padding") { + DEFAULT_RSA_VARIANT must be("RSA/ECB/PKCS1Padding") + } + } + + describe("the signature algorithm") { + it("must be SHA256withRSA") { + SIGNATURE_ALGORITHM must be("SHA512withRSA") + } + } + + describe("generateAESKeyAndIV") { + it("must generate an aes key and an init vector") { + val response = generateAESKeyAndIV() + response._1.getAlgorithm must be("AES") + response._2.length must be(DEFAULT_AES_KEY_LENGTH / 8) + } + } + + describe("generateAESKeyFromParameters") { + it("must generate the correct key") { + val response = generateAESKeyAndIV() + val key = response._1 + val generated = generateAESKeyFromParameters(key.getEncoded) + + generated.getAlgorithm must be(key.getAlgorithm) + generated.getEncoded must be(key.getEncoded) + } + } + + describe("generateRSAKeyPair") { + it("must generate a RSA key pair") { + val keyPair = generateRSAKeyPair(512) + keyPair.getPrivate.getAlgorithm must be("RSA") + keyPair.getPublic.getAlgorithm must be("RSA") + } + } + + describe("generateRSAKeyFromParameters") { + describe("if an illegal key type is desired") { + it("must throw an exception") { + val keyPair = generateRSAKeyPair(512) + val keyFactory = KeyFactory.getInstance("RSA") + val keySpec = keyFactory.getKeySpec(keyPair.getPrivate, classOf[RSAPrivateKeySpec]) + + an[IllegalArgumentException] must be thrownBy generateRSAKeyFromParameters( + keySpec.getModulus.toByteArray, + keySpec.getPrivateExponent.toByteArray, + classOf[String] + ) + } + } + + describe("if a private key is desired") { + it("must return a private key") { + val keyPair = generateRSAKeyPair(512) + val keyFactory = KeyFactory.getInstance("RSA") + val keySpec = keyFactory.getKeySpec(keyPair.getPrivate, classOf[RSAPrivateKeySpec]) + val response = generateRSAKeyFromParameters(keySpec.getModulus.toByteArray, + keySpec.getPrivateExponent.toByteArray, + classOf[RSAPrivateKey]) + response.getAlgorithm must be(keyPair.getPrivate.getAlgorithm) + val responseKeySpec = keyFactory.getKeySpec(response, classOf[RSAPrivateKeySpec]) + responseKeySpec.getModulus.compareTo(keySpec.getModulus) must be(0) + responseKeySpec.getPrivateExponent.compareTo(keySpec.getPrivateExponent) must be(0) + } + } + + describe("if a public key is desired") { + it("must return a public key") { + val keyPair = generateRSAKeyPair(512) + val keyFactory = KeyFactory.getInstance("RSA") + val keySpec = keyFactory.getKeySpec(keyPair.getPublic, classOf[RSAPublicKeySpec]) + val response = generateRSAKeyFromParameters(keySpec.getModulus.toByteArray, + keySpec.getPublicExponent.toByteArray, + classOf[RSAPublicKey]) + response.getAlgorithm must be(keyPair.getPublic.getAlgorithm) + val responseKeySpec = keyFactory.getKeySpec(response, classOf[RSAPublicKeySpec]) + responseKeySpec.getModulus.compareTo(keySpec.getModulus) must be(0) + responseKeySpec.getPublicExponent.compareTo(keySpec.getPublicExponent) must be(0) + } + } + } + + describe("getAESCipher") { + it("must return an AES cipher using the default AES variant") { + getAESCipher.getAlgorithm must be(Cipher.getInstance(DEFAULT_AES_VARIANT).getAlgorithm) + } + } + + describe("getRSACipher") { + it("must return a RSA cipher using the default RSA variant") { + getRSACipher.getAlgorithm must be(Cipher.getInstance(DEFAULT_RSA_VARIANT).getAlgorithm) + } + } + + describe("encrypt") { + describe(s"using an AES key with $DEFAULT_AES_KEY_LENGTH bits") { + it("must encrypt the given data") { + val keyAndIV = generateAESKeyAndIV() + val sourceData = "My voice is my passport, verify me!" + + encrypt(sourceData.getBytes, getAESCipher, keyAndIV._1, Option(keyAndIV._2)) match { + case -\/(error) ⇒ fail(error) + case \/-(success) ⇒ success.length must be > 0 + } + } + + it("must encrypt a large amount of data") { + val keyAndIV = generateAESKeyAndIV() + val sourceData = scala.util.Random.alphanumeric.take(1024).mkString + + encrypt(sourceData.getBytes, getAESCipher, keyAndIV._1, Option(keyAndIV._2)) match { + case -\/(error) ⇒ fail(error) + case \/-(success) ⇒ success.length must be > 0 + } + } + } + + describe(s"using a RSA key with $DEFAULT_RSA_KEY_LENGTH bits") { + it("must encrypt the given data") { + val keyPair = generateRSAKeyPair() + val sourceData = "My voice is my passport, verify me!" + + encrypt(sourceData.getBytes, getRSACipher, keyPair.getPublic, None) match { + case -\/(error) ⇒ fail(error) + case \/-(success) ⇒ success.length must be > 0 + } + } + + it(s"must not encrypt more than ${DEFAULT_RSA_KEY_LENGTH / 8 - 11} bytes") { + val keyPair = generateRSAKeyPair() + val sourceData = + scala.util.Random.alphanumeric.take(DEFAULT_RSA_KEY_LENGTH / 8 - 10).mkString + + val response = encrypt(sourceData.getBytes, getRSACipher, keyPair.getPublic, None) + response.swap.getOrElse(new RuntimeException("An unexpected exception")) mustBe a[ + IllegalBlockSizeException + ] + } + } + } + + describe("decrypt") { + describe(s"using an AES key with $DEFAULT_AES_KEY_LENGTH bits") { + it("must decrypt the given data") { + val keyAndIV = generateAESKeyAndIV() + val sourceData = "My voice is my passport, verify me!" + + encrypt(sourceData.getBytes, getAESCipher, keyAndIV._1, Option(keyAndIV._2)) fold ( + error ⇒ fail(error), + encryptedData ⇒ { + val decoded = java.util.Base64.getDecoder.decode(encryptedData) + decrypt(decoded, getAESCipher, keyAndIV._1, Option(keyAndIV._2)) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ new String(success) must be(sourceData) + } + } + ) + } + } + + describe(s"using a RSA ky with $DEFAULT_RSA_KEY_LENGTH bits") { + it("must decrypt the given data") { + val keyPair = generateRSAKeyPair() + val sourceData = "My voice is my passport, verify me!" + + encrypt(sourceData.getBytes, getRSACipher, keyPair.getPublic, None) fold ( + error ⇒ fail(error), + encryptedData ⇒ { + val decoded = java.util.Base64.getDecoder.decode(encryptedData) + decrypt(decoded, getRSACipher, keyPair.getPrivate, None) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ new String(success) must be(sourceData) + } + } + ) + } + } + } + + describe("sign") { + it("must generate a signature") { + val keyPair = generateRSAKeyPair() + val sourceData = "My voice is my passport, verify me!" + + sign(sourceData.getBytes, keyPair.getPrivate) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + val signature = java.util.Base64.getDecoder.decode(success) + signature.length must be(256) + validate(sourceData.getBytes, signature, keyPair.getPublic) must be(right = true) + } + } + + it("must generate a signature for large data") { + val keyPair = generateRSAKeyPair() + val sourceData = scala.util.Random.alphanumeric.take(4096).mkString + + sign(sourceData.getBytes, keyPair.getPrivate) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + val signature = java.util.Base64.getDecoder.decode(success) + signature.length must be(256) + validate(sourceData.getBytes, signature, keyPair.getPublic) must be(right = true) + } + } + + it("must generate a signature for a license") { + val keyPair = generateRSAKeyPair() + val license = + TenseiLicense.createProfessionalLicense("TEST-LICENSE", LocalDate.now().plusYears(1L)) + val licenseString = license.asJson.nospaces + + sign(licenseString.getBytes, keyPair.getPrivate) match { + case -\/(failure) ⇒ fail(failure) + case \/-(success) ⇒ + val messageString = + List(licenseString, new String(success)).mkString("\n----- SIGNATURE -----\n") + val messageParts = messageString.split("\n----- SIGNATURE -----\n") + val signature = java.util.Base64.getDecoder.decode(messageParts(1).getBytes) + signature.length must be(256) + validate(messageParts(0).getBytes, signature, keyPair.getPublic) must be(right = true) + } + } + } + + describe("validate") { + it("must return true for a valid signature") { + val keyPair = generateRSAKeyPair() + val sourceData = "My voice is my passport, verify me!" + + val signature = sign(sourceData.getBytes, keyPair.getPrivate).getOrElse(Array.empty[Byte]) + val decodedSignature = java.util.Base64.getDecoder.decode(signature) + decodedSignature.length must be(256) + + withClue("The signature must be valid!")( + validate(sourceData.getBytes, decodedSignature, keyPair.getPublic) must be(right = true) + ) + } + + it("must return false for an invalid signature") { + val keyPair = generateRSAKeyPair() + val sourceData = "My voice is my passport, verify me!" + val signature = scala.util.Random.alphanumeric.take(256).mkString + + withClue("The signature must not be valid!")( + validate(sourceData.getBytes, signature.getBytes, keyPair.getPublic) must be( + right = false + ) + ) + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/server/ChefDeCuisineStateTest.scala b/src/test/scala/com/wegtam/tensei/server/ChefDeCuisineStateTest.scala new file mode 100644 index 0000000..431b7ad --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/server/ChefDeCuisineStateTest.scala @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class ChefDeCuisineStateTest extends DefaultSpec { + describe("ChefDeCuisineState") { + describe("CodecJson") { + describe("decode") { + it("must decode all known states correctly") { + Parse.decodeOption[ChefDeCuisineState](""""Booting"""").get must be( + ChefDeCuisineState.Booting + ) + Parse.decodeOption[ChefDeCuisineState](""""Initializing"""").get must be( + ChefDeCuisineState.Initializing + ) + Parse.decodeOption[ChefDeCuisineState](""""Running"""").get must be( + ChefDeCuisineState.Running + ) + } + + it("must throw an IllegalArgumentException upon an unknown state") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[ChefDeCuisineState](""""SomeUnknownState"""") + ) + } + } + + describe("encode") { + it("must encode all known states correctly") { + ChefDeCuisineState.Booting.asJson.nospaces must be(""""Booting"""") + ChefDeCuisineState.Initializing.asJson.nospaces must be(""""Initializing"""") + ChefDeCuisineState.Running.asJson.nospaces must be(""""Running"""") + } + } + } + } +} diff --git a/src/test/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModesTest.scala b/src/test/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModesTest.scala new file mode 100644 index 0000000..134f164 --- /dev/null +++ b/src/test/scala/com/wegtam/tensei/server/suggesters/MappingSuggesterModesTest.scala @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2014 - 2017 Contributors as noted in the AUTHORS.md file + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.wegtam.tensei.server.suggesters + +import argonaut._, Argonaut._ + +import com.wegtam.tensei.DefaultSpec + +class MappingSuggesterModesTest extends DefaultSpec { + describe("MappingSuggesterModes") { + describe("CodecJson") { + describe("decode") { + it("must decode all known modes correctly") { + Parse.decodeOption[MappingSuggesterModes](""""Simple"""").get must be( + MappingSuggesterModes.Simple + ) + Parse.decodeOption[MappingSuggesterModes](""""SimpleWithTransformers"""").get must be( + MappingSuggesterModes.SimpleWithTransformers + ) + Parse.decodeOption[MappingSuggesterModes](""""SimpleSemantics"""").get must be( + MappingSuggesterModes.SimpleSemantics + ) + Parse + .decodeOption[MappingSuggesterModes](""""SimpleSemanticsWithTransformers"""") + .get must be(MappingSuggesterModes.SimpleSemanticsWithTransformers) + Parse.decodeOption[MappingSuggesterModes](""""AdvancedSemantics"""").get must be( + MappingSuggesterModes.AdvancedSemantics + ) + } + + it("must throw an IllegalArgumentException upon an unknown mode") { + an[IllegalArgumentException] shouldBe thrownBy( + Parse.decode[MappingSuggesterModes](""""SomeUnknownMode"""") + ) + } + } + + describe("encode") { + it("must encode all known modes correctly") { + MappingSuggesterModes.Simple.asJson.nospaces must be(""""Simple"""") + MappingSuggesterModes.SimpleWithTransformers.asJson.nospaces must be( + """"SimpleWithTransformers"""" + ) + MappingSuggesterModes.SimpleSemantics.asJson.nospaces must be(""""SimpleSemantics"""") + MappingSuggesterModes.SimpleSemanticsWithTransformers.asJson.nospaces must be( + """"SimpleSemanticsWithTransformers"""" + ) + MappingSuggesterModes.AdvancedSemantics.asJson.nospaces must be( + """"AdvancedSemantics"""" + ) + } + } + } + } +}