From 9a3869d9ed2fd59e9deed383f46429b3762366b7 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 7 Aug 2024 20:38:24 -0300 Subject: [PATCH 1/2] Issue #1586: Drop ClearNLP module --- dkpro-core-clearnlp-asl/src/scripts/build.xml | 1160 +++--- dkpro-core-opennlp-asl/src/scripts/build.xml | 3250 ++++++++--------- 2 files changed, 2205 insertions(+), 2205 deletions(-) diff --git a/dkpro-core-clearnlp-asl/src/scripts/build.xml b/dkpro-core-clearnlp-asl/src/scripts/build.xml index 941045a528..0679428309 100644 --- a/dkpro-core-clearnlp-asl/src/scripts/build.xml +++ b/dkpro-core-clearnlp-asl/src/scripts/build.xml @@ -1,581 +1,581 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dkpro-core-opennlp-asl/src/scripts/build.xml b/dkpro-core-opennlp-asl/src/scripts/build.xml index 22f7a1c2b8..881e6617e1 100644 --- a/dkpro-core-opennlp-asl/src/scripts/build.xml +++ b/dkpro-core-opennlp-asl/src/scripts/build.xml @@ -1,1626 +1,1626 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 4c48e207d497fc472c51686b7be8601d6b8ee9b3 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 7 Aug 2024 20:42:29 -0300 Subject: [PATCH 2/2] #1586 - Drop ClearNLP module - Removed module - Switch out lemmatizer in Web1T test --- dkpro-core-asl/pom.xml | 1 - dkpro-core-bom-asl/pom.xml | 5 - dkpro-core-clearnlp-asl/LICENSE.txt | 202 -------- dkpro-core-clearnlp-asl/pom.xml | 287 ----------- .../core/clearnlp/ClearNlpLemmatizer.java | 177 ------- .../dkpro/core/clearnlp/ClearNlpParser.java | 271 ----------- .../core/clearnlp/ClearNlpPosTagger.java | 309 ------------ .../core/clearnlp/ClearNlpSegmenter.java | 152 ------ .../clearnlp/ClearNlpSemanticRoleLabeler.java | 444 ------------------ .../clearnlp/lib/parser-default-variants.map | 1 - .../core/clearnlp/ClearNlpLemmatizerTest.java | 70 --- .../core/clearnlp/ClearNlpParserTest.java | 114 ----- .../core/clearnlp/ClearNlpPosTaggerTest.java | 91 ---- .../core/clearnlp/ClearNlpSegmenterTest.java | 77 --- .../ClearNlpSemanticRoleLabelerTest.java | 130 ----- .../src/test/resources/log4j2.xml | 15 - dkpro-core-io-web1t-asl/pom.xml | 27 +- .../dkpro/core/io/web1t/Web1TWriterTest.java | 122 +++-- .../src/test/resources/data/test1.txt | 1 + .../src/test/resources/data/test2.txt | 1 + .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../src/test/resources/test1.txt | 1 - .../src/test/resources/test2.txt | 1 - 23 files changed, 61 insertions(+), 2438 deletions(-) delete mode 100644 dkpro-core-clearnlp-asl/LICENSE.txt delete mode 100644 dkpro-core-clearnlp-asl/pom.xml delete mode 100644 dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpLemmatizer.java delete mode 100644 dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpParser.java delete mode 100644 dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpPosTagger.java delete mode 100644 dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSegmenter.java delete mode 100644 dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabeler.java delete mode 100644 dkpro-core-clearnlp-asl/src/main/resources/org/dkpro/core/clearnlp/lib/parser-default-variants.map delete mode 100644 dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpLemmatizerTest.java delete mode 100644 dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpParserTest.java delete mode 100644 dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpPosTaggerTest.java delete mode 100644 dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSegmenterTest.java delete mode 100644 dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabelerTest.java delete mode 100644 dkpro-core-clearnlp-asl/src/test/resources/log4j2.xml create mode 100644 dkpro-core-io-web1t-asl/src/test/resources/data/test1.txt create mode 100644 dkpro-core-io-web1t-asl/src/test/resources/data/test2.txt rename dkpro-core-io-web1t-asl/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 dkpro-core-io-web1t-asl/src/test/resources/test1.txt delete mode 100644 dkpro-core-io-web1t-asl/src/test/resources/test2.txt diff --git a/dkpro-core-asl/pom.xml b/dkpro-core-asl/pom.xml index 586f04bafc..a5f82bf462 100644 --- a/dkpro-core-asl/pom.xml +++ b/dkpro-core-asl/pom.xml @@ -106,7 +106,6 @@ ../dkpro-core-castransformation-asl ../dkpro-core-cisstem-asl - ../dkpro-core-clearnlp-asl ../dkpro-core-commonscodec-asl ../dkpro-core-decompounding-asl ../dkpro-core-dictionaryannotator-asl diff --git a/dkpro-core-bom-asl/pom.xml b/dkpro-core-bom-asl/pom.xml index a99e1bb249..a31e044063 100644 --- a/dkpro-core-bom-asl/pom.xml +++ b/dkpro-core-bom-asl/pom.xml @@ -149,11 +149,6 @@ dkpro-core-cisstem-asl 3.0.0-SNAPSHOT - - org.dkpro.core - dkpro-core-clearnlp-asl - 3.0.0-SNAPSHOT - org.dkpro.core dkpro-core-commonscodec-asl diff --git a/dkpro-core-clearnlp-asl/LICENSE.txt b/dkpro-core-clearnlp-asl/LICENSE.txt deleted file mode 100644 index d645695673..0000000000 --- a/dkpro-core-clearnlp-asl/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dkpro-core-clearnlp-asl/pom.xml b/dkpro-core-clearnlp-asl/pom.xml deleted file mode 100644 index e9e6f03970..0000000000 --- a/dkpro-core-clearnlp-asl/pom.xml +++ /dev/null @@ -1,287 +0,0 @@ - - - 4.0.0 - - org.dkpro.core - dkpro-core-asl - 3.0.0-SNAPSHOT - ../dkpro-core-asl - - - dkpro-core-clearnlp-asl - jar - DKPro Core ASL - Clear NLP (v ${clearnlp.version}) - https://dkpro.github.io/dkpro-core/ - - - 2.0.2 - - - - - com.clearnlp - clearnlp - ${clearnlp.version} - - - org.apache.uima - uimafit-core - - - org.dkpro.core - dkpro-core-api-lexmorph-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-syntax-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-semantics-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-segmentation-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-metadata-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-resources-asl - ${project.version} - - - org.dkpro.core - dkpro-core-api-parameter-asl - ${project.version} - - - eu.openminted.share.annotations - omtd-share-annotations-api - - - org.apache.commons - commons-lang3 - test - - - org.dkpro.core - dkpro-core-testing-asl - ${project.version} - test - - - org.dkpro.core - dkpro-core-opennlp-asl - ${project.version} - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.opennlp-model-tagger-en-maxent - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-dictionary-en-default - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-lemma-en-default - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-segmenter-en-default - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-mayo - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-ontonotes - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-mayo - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-ontonotes - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-ontonotes - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-mayo - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-ontonotes - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-mayo - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-ontonotes - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-mayo - test - - - commons-io - commons-io - - - org.apache.uima - uimaj-core - - - - - - - org.dkpro.core - dkpro-core-opennlp-asl - ${project.version} - pom - import - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-dictionary-en-default - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-lemma-en-default - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-segmenter-en-default - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-mayo - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-ontonotes - 20131128.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-mayo - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-ontonotes - 20131128.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-ontonotes - 20131128.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-mayo - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-ontonotes - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-mayo - 20131111.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-ontonotes - 20131128.0 - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-mayo - 20131111.0 - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.opennlp-model-tagger-en-maxent - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-dictionary-en-default - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-lemma-en-default - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-segmenter-en-default - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-mayo - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-parser-en-ontonotes - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-mayo - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-tagger-en-ontonotes - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-ontonotes - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-pred-en-mayo - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-ontonotes - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-role-en-mayo - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-ontonotes - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-srl-en-mayo - - - - - - - \ No newline at end of file diff --git a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpLemmatizer.java b/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpLemmatizer.java deleted file mode 100644 index 729a65df63..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpLemmatizer.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.apache.uima.fit.util.JCasUtil.selectCovered; - -import java.io.InputStream; -import java.util.List; - -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.component.JCasAnnotator_ImplBase; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CasConfigurableProviderBase; -import org.dkpro.core.api.resources.ModelProviderBase; - -import com.clearnlp.component.AbstractComponent; -import com.clearnlp.component.morph.DefaultMPAnalyzer; -import com.clearnlp.component.morph.EnglishMPAnalyzer; -import com.clearnlp.dependency.DEPNode; -import com.clearnlp.dependency.DEPTree; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import eu.openminted.share.annotations.api.Component; -import eu.openminted.share.annotations.api.DocumentationResource; -import eu.openminted.share.annotations.api.constants.OperationType; - -/** - * Lemmatizer using Clear NLP. - */ -@Component(OperationType.LEMMATIZER) -@ResourceMetaData(name = "ClearNLP Lemmatizer") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@TypeCapability( - inputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS" }, - outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma" } -) -public class ClearNlpLemmatizer - extends JCasAnnotator_ImplBase -{ - - /** - * Use this language instead of the document language to resolve the model. - */ - public static final String PARAM_LANGUAGE = ComponentParameters.PARAM_LANGUAGE; - @ConfigurationParameter(name = PARAM_LANGUAGE, mandatory = false, defaultValue = "en") - protected String language; - - /** - * Override the default variant used to locate the model. - */ - public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; - @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) - protected String variant; - - /** - * URI of the model artifact. This can be used to override the default model resolving - * mechanism and directly address a particular model. - * - *

The URI format is {@code mvn:${groupId}:${artifactId}:${version}}. Remember to set - * the variant parameter to match the artifact. If the artifact contains the model in - * a non-default location, you also have to specify the model location parameter, e.g. - * {@code classpath:/model/path/in/artifact/model.bin}.

- */ - public static final String PARAM_MODEL_ARTIFACT_URI = - ComponentParameters.PARAM_MODEL_ARTIFACT_URI; - @ConfigurationParameter(name = PARAM_MODEL_ARTIFACT_URI, mandatory = false) - protected String modelArtifactUri; - - /** - * Load the model from this location instead of locating the model automatically. - */ - public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; - @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = false) - protected String modelLocation; - - private CasConfigurableProviderBase modelProvider; - - @Override - public void initialize(UimaContext aContext) - throws ResourceInitializationException - { - super.initialize(aContext); - - modelProvider = new ModelProviderBase(this, "clearnlp", "lemma") - { - { - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/lemma-${language}-${variant}.properties"); - } - - @Override - protected AbstractComponent produceResource(InputStream aStream) - throws Exception - { - String lang = getAggregatedProperties().getProperty(LANGUAGE); - AbstractComponent lemmatizer; - if (lang.equals("en")) { - lemmatizer = new EnglishMPAnalyzer(aStream); - } - else { - lemmatizer = new DefaultMPAnalyzer(); - } - return lemmatizer; - } - }; - } - - @Override - public void process(JCas aJCas) - throws AnalysisEngineProcessException - { - - modelProvider.configure(aJCas.getCas()); - AbstractComponent analyzer = modelProvider.getResource(); - - // Iterate over all sentences - for (Sentence sentence : select(aJCas, Sentence.class)) { - List tokens = selectCovered(aJCas, Token.class, sentence); - - DEPTree tree = new DEPTree(); - - // Generate input format required by analyzer - for (int i = 0; i < tokens.size(); i++) { - Token t = tokens.get(i); - DEPNode node = new DEPNode(i + 1, tokens.get(i).getText()); - node.pos = t.getPos().getPosValue(); - tree.add(node); - } - - analyzer.process(tree); - - int i = 0; - for (Token t : tokens) { - DEPNode node = tree.get(i + 1); - String lemmaString = node.lemma; - if (lemmaString == null) { - lemmaString = t.getText(); - } - Lemma l = new Lemma(aJCas, t.getBegin(), t.getEnd()); - l.setValue(lemmaString); - l.addToIndexes(); - - t.setLemma(l); - i++; - } - } - } -} diff --git a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpParser.java b/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpParser.java deleted file mode 100644 index aaf120252a..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpParser.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.commons.io.IOUtils.closeQuietly; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.apache.uima.fit.util.JCasUtil.selectCovered; -import static org.apache.uima.util.Level.INFO; -import static org.apache.uima.util.Level.WARNING; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.net.URL; -import java.util.List; -import java.util.Properties; -import java.util.zip.GZIPInputStream; - -import org.apache.commons.io.FileUtils; -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_component.AnalysisComponent; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.component.JCasAnnotator_ImplBase; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.metadata.SingletonTagset; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CasConfigurableProviderBase; -import org.dkpro.core.api.resources.ModelProviderBase; - -import com.clearnlp.classification.model.StringModel; -import com.clearnlp.component.dep.AbstractDEPParser; -import com.clearnlp.dependency.DEPNode; -import com.clearnlp.dependency.DEPTree; -import com.clearnlp.nlp.NLPGetter; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.DependencyFlavor; -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.ROOT; -import eu.openminted.share.annotations.api.Component; -import eu.openminted.share.annotations.api.DocumentationResource; -import eu.openminted.share.annotations.api.constants.OperationType; - -/** - * CLEAR parser annotator. - */ -@Component(OperationType.DEPENDENCY_PARSER) -@ResourceMetaData(name = "ClearNLP Parser") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@TypeCapability( - inputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma" }, - outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency" }) -public class ClearNlpParser - extends JCasAnnotator_ImplBase -{ - /** - * Write the tag set(s) to the log when a model is loaded. - */ - public static final String PARAM_PRINT_TAGSET = ComponentParameters.PARAM_PRINT_TAGSET; - @ConfigurationParameter(name = PARAM_PRINT_TAGSET, mandatory = true, defaultValue = "false") - protected boolean printTagSet; - - /** - * Use this language instead of the document language to resolve the model. - */ - public static final String PARAM_LANGUAGE = ComponentParameters.PARAM_LANGUAGE; - @ConfigurationParameter(name = PARAM_LANGUAGE, mandatory = false) - protected String language; - - /** - * Variant of a model the model. Used to address a specific model if here are multiple models - * for one language. - */ - public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; - @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) - protected String variant; - - /** - * URI of the model artifact. This can be used to override the default model resolving - * mechanism and directly address a particular model. - * - *

The URI format is {@code mvn:${groupId}:${artifactId}:${version}}. Remember to set - * the variant parameter to match the artifact. If the artifact contains the model in - * a non-default location, you also have to specify the model location parameter, e.g. - * {@code classpath:/model/path/in/artifact/model.bin}.

- */ - public static final String PARAM_MODEL_ARTIFACT_URI = - ComponentParameters.PARAM_MODEL_ARTIFACT_URI; - @ConfigurationParameter(name = PARAM_MODEL_ARTIFACT_URI, mandatory = false) - protected String modelArtifactUri; - - /** - * Location from which the model is read. - */ - public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; - @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = false) - protected String modelLocation; - - private File workingDir; - - private CasConfigurableProviderBase parserProvider; - - @Override - public void initialize(UimaContext context) - throws ResourceInitializationException - { - super.initialize(context); - - parserProvider = new ModelProviderBase(this, "clearnlp", "parser") - { - { - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/parser-${language}-${variant}.properties"); - } - - @Override - protected AbstractDEPParser produceResource(URL aUrl) - throws IOException - { - InputStream is = null; - BufferedInputStream bis = null; - ObjectInputStream ois = null; - GZIPInputStream gis = null; - - try { - is = aUrl.openStream(); - String language = getAggregatedProperties().getProperty(LANGUAGE); - gis = new GZIPInputStream(is); - bis = new BufferedInputStream(gis); - ois = new ObjectInputStream(bis); - AbstractDEPParser parser = NLPGetter.getDEPParser(ois, language); - Properties metadata = getResourceMetaData(); - - SingletonTagset depTags = new SingletonTagset(Dependency.class, - metadata.getProperty("dependency.tagset")); - - try { - for (StringModel model : parser.getModels()) { - for (String label : model.getLabels()) { - String[] fields = label.split("_"); - if (fields.length == 3) { - depTags.add(fields[2]); - } - // else { - // getContext().getLogger().log(WARNING, - // "Unknown label format: [" + label + "]"); - // } - } - } - } - catch (Exception e) { - getContext().getLogger().log(WARNING, "Unable to find tagset information."); - } - - addTagset(depTags); - - if (printTagSet) { - getContext().getLogger().log(INFO, getTagset().toString()); - } - - return parser; - } - catch (Exception e) { - throw new IOException(e); - } - finally { - closeQuietly(ois); - closeQuietly(bis); - closeQuietly(gis); - closeQuietly(is); - } - } - }; - } - - /** - * @see AnalysisComponent#collectionProcessComplete() - */ - @Override - public void collectionProcessComplete() - throws AnalysisEngineProcessException - { - if ((workingDir != null) && workingDir.isDirectory()) { - FileUtils.deleteQuietly(workingDir); - } - } - - @Override - public void process(JCas aJCas) - throws AnalysisEngineProcessException - { - parserProvider.configure(aJCas.getCas()); - - // Iterate over all sentences - for (Sentence sentence : select(aJCas, Sentence.class)) { - List tokens = selectCovered(aJCas, Token.class, sentence); - - DEPTree tree = new DEPTree(); - - // Generate input format required by parser - for (int i = 0; i < tokens.size(); i++) { - Token t = tokens.get(i); - DEPNode node = new DEPNode(i + 1, tokens.get(i).getText()); - node.pos = t.getPos().getPosValue(); - if (t.getLemma() != null) { - node.lemma = t.getLemma().getValue(); - } - tree.add(node); - } - - // Parse sentence - AbstractDEPParser parser = parserProvider.getResource(); - parser.process(tree); - - for (int i = 1; i < tree.size(); i++) { - DEPNode node = tree.get(i); - - if (node.hasHead()) { - if (node.getHead().id != 0) { - Dependency dep = new Dependency(aJCas); - dep.setGovernor(tokens.get(node.getHead().id - 1)); - dep.setDependent(tokens.get(node.id - 1)); - dep.setDependencyType(node.getLabel()); - dep.setBegin(dep.getDependent().getBegin()); - dep.setEnd(dep.getDependent().getEnd()); - dep.setFlavor(DependencyFlavor.BASIC); - dep.addToIndexes(); - } - else { - Dependency dep = new ROOT(aJCas); - dep.setGovernor(tokens.get(node.id - 1)); - dep.setDependent(tokens.get(node.id - 1)); - dep.setDependencyType("ROOT"); - dep.setBegin(dep.getDependent().getBegin()); - dep.setEnd(dep.getDependent().getEnd()); - dep.setFlavor(DependencyFlavor.BASIC); - dep.addToIndexes(); - } - } - } - } - } -} diff --git a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpPosTagger.java b/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpPosTagger.java deleted file mode 100644 index 82cb81450d..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpPosTagger.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static java.util.Arrays.asList; -import static org.apache.commons.io.IOUtils.closeQuietly; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.apache.uima.fit.util.JCasUtil.selectCovered; -import static org.apache.uima.fit.util.JCasUtil.toText; -import static org.apache.uima.util.Level.INFO; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.util.List; -import java.util.zip.GZIPInputStream; - -import org.apache.commons.io.IOUtils; -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.cas.CAS; -import org.apache.uima.cas.Type; -import org.apache.uima.fit.component.JCasAnnotator_ImplBase; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.lexmorph.pos.POSUtils; -import org.dkpro.core.api.metadata.SingletonTagset; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CasConfigurableProviderBase; -import org.dkpro.core.api.resources.CasConfigurableStreamProviderBase; -import org.dkpro.core.api.resources.MappingProvider; -import org.dkpro.core.api.resources.MappingProviderFactory; -import org.dkpro.core.api.resources.ModelProviderBase; - -import com.clearnlp.classification.model.StringModel; -import com.clearnlp.component.AbstractComponent; -import com.clearnlp.component.morph.EnglishMPAnalyzer; -import com.clearnlp.component.pos.AbstractPOSTagger; -import com.clearnlp.component.pos.DefaultPOSTagger; -import com.clearnlp.component.pos.EnglishPOSTagger; -import com.clearnlp.dependency.DEPTree; -import com.clearnlp.nlp.NLPGetter; - -import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import eu.openminted.share.annotations.api.Component; -import eu.openminted.share.annotations.api.DocumentationResource; -import eu.openminted.share.annotations.api.constants.OperationType; - -/** - * Part-of-Speech annotator using Clear NLP. Requires {@link Sentence}s to be annotated before. - */ -@Component(OperationType.PART_OF_SPEECH_TAGGER) -@ResourceMetaData(name = "ClearNLP POS-Tagger") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@TypeCapability( - inputs = { "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence" }, - outputs = { "de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS" }) -public class ClearNlpPosTagger - extends JCasAnnotator_ImplBase -{ - /** - * Use this language instead of the document language to resolve the model. - */ - public static final String PARAM_LANGUAGE = ComponentParameters.PARAM_LANGUAGE; - @ConfigurationParameter(name = PARAM_LANGUAGE, mandatory = false) - protected String language; - - - /** - * Override the default variant used to locate the dictionary. - */ - public static final String PARAM_DICT_VARIANT = "dictVariant"; - @ConfigurationParameter(name = PARAM_DICT_VARIANT, mandatory = false) - protected String dictVariant; - - /** - * Load the dictionary from this location instead of locating the dictionary automatically. - */ - public static final String PARAM_DICT_LOCATION = "dictLocation"; - @ConfigurationParameter(name = PARAM_DICT_LOCATION, mandatory = false) - protected String dictLocation; - - /** - * Override the default variant used to locate the pos-tagging model. - */ - public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; - @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) - protected String posVariant; - - /** - * URI of the model artifact. This can be used to override the default model resolving - * mechanism and directly address a particular model. - * - *

The URI format is {@code mvn:${groupId}:${artifactId}:${version}}. Remember to set - * the variant parameter to match the artifact. If the artifact contains the model in - * a non-default location, you also have to specify the model location parameter, e.g. - * {@code classpath:/model/path/in/artifact/model.bin}.

- */ - public static final String PARAM_MODEL_ARTIFACT_URI = - ComponentParameters.PARAM_MODEL_ARTIFACT_URI; - @ConfigurationParameter(name = PARAM_MODEL_ARTIFACT_URI, mandatory = false) - protected String modelArtifactUri; - - /** - * Load the model from this location instead of locating the pos-tagging model automatically. - */ - public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; - @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = false) - protected String posModelLocation; - - /** - * Enable/disable type mapping. - */ - public static final String PARAM_MAPPING_ENABLED = ComponentParameters.PARAM_MAPPING_ENABLED; - @ConfigurationParameter(name = PARAM_MAPPING_ENABLED, mandatory = true, defaultValue = - ComponentParameters.DEFAULT_MAPPING_ENABLED) - protected boolean mappingEnabled; - - /** - * Load the part-of-speech tag to UIMA type mapping from this location instead of locating the - * mapping automatically. - */ - public static final String PARAM_POS_MAPPING_LOCATION = - ComponentParameters.PARAM_POS_MAPPING_LOCATION; - @ConfigurationParameter(name = PARAM_POS_MAPPING_LOCATION, mandatory = false) - protected String posMappingLocation; - - /** - * Log the tag set(s) when a model is loaded. - */ - public static final String PARAM_PRINT_TAGSET = ComponentParameters.PARAM_PRINT_TAGSET; - @ConfigurationParameter(name = PARAM_PRINT_TAGSET, mandatory = true, defaultValue = "false") - protected boolean printTagSet; - - private CasConfigurableProviderBase dictModelProvider; - private CasConfigurableProviderBase posTaggingModelProvider; - private MappingProvider posMappingProvider; - - @Override - public void initialize(UimaContext aContext) - throws ResourceInitializationException - { - super.initialize(aContext); - - dictModelProvider = new CasConfigurableStreamProviderBase() - { - { - setContextObject(ClearNlpPosTagger.this); - - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(ARTIFACT_ID, "${groupId}.clearnlp-model-dictionary-${language}-${variant}"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/dictionary-${language}-${variant}.properties"); - setDefaultVariantsLocation("${package}/lib/dictionary-default-variants.map"); - setDefault(VARIANT, "default"); - - setOverride(ARTIFACT_URI, modelArtifactUri); - setOverride(LOCATION, dictLocation); - setOverride(LANGUAGE, language); - setOverride(VARIANT, dictVariant); - } - - @Override - protected InputStream produceResource(InputStream aStream) - throws Exception - { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - IOUtils.copy(aStream, os); - byte[] array = os.toByteArray(); - InputStream is = new ByteArrayInputStream(array); - return is; - } - }; - - posTaggingModelProvider = new ModelProviderBase(this, "clearnlp", "tagger") - { - { - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(VARIANT, "ontonotes"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/tagger-${language}-${variant}.properties"); - } - - @Override - protected AbstractPOSTagger produceResource(InputStream aStream) - throws Exception - { - - BufferedInputStream bis = null; - ObjectInputStream ois = null; - GZIPInputStream gis = null; - - try { - gis = new GZIPInputStream(aStream); - bis = new BufferedInputStream(gis); - ois = new ObjectInputStream(bis); - - String language = getAggregatedProperties().getProperty(LANGUAGE); - AbstractPOSTagger tagger; - if (language.equals("en")) { - tagger = new DkproPosTagger(ois); - } - else { - tagger = new DefaultPOSTagger(ois); - } - - SingletonTagset tags = new SingletonTagset(POS.class, getResourceMetaData() - .getProperty(("pos.tagset"))); - - for (StringModel model : tagger.getModels()) { - tags.addAll(asList(model.getLabels())); - } - addTagset(tags, true); - - if (printTagSet) { - getContext().getLogger().log(INFO, getTagset().toString()); - } - - return tagger; - } - catch (Exception e) { - throw new IOException(e); - } - finally { - closeQuietly(ois); - closeQuietly(bis); - closeQuietly(gis); - } - } - - }; - - posMappingProvider = MappingProviderFactory.createPosMappingProvider(this, - posMappingLocation, language, posTaggingModelProvider); - } - - @Override - public void process(JCas aJCas) - throws AnalysisEngineProcessException - { - CAS cas = aJCas.getCas(); - dictModelProvider.configure(cas); - posTaggingModelProvider.configure(cas); - posMappingProvider.configure(cas); - - for (Sentence sentence : select(aJCas, Sentence.class)) { - List tokens = selectCovered(aJCas, Token.class, sentence); - List tokenTexts = asList(toText(tokens).toArray(new String[tokens.size()])); - - DEPTree tree = NLPGetter.toDEPTree(tokenTexts); - - AbstractComponent tagger = posTaggingModelProvider.getResource(); - tagger.process(tree); - - String[] posTags = tree.getPOSTags(); - - int i = 0; - for (Token t : tokens) { - String tag = posTags[i + 1]; - Type posTag = posMappingProvider.getTagType(tag != null ? tag.intern() : null); - POS posAnno = (POS) cas.createAnnotation(posTag, t.getBegin(), t.getEnd()); - posAnno.setPosValue(tag); - POSUtils.assignCoarseValue(posAnno); - posAnno.addToIndexes(); - t.setPos(posAnno); - i++; - } - } - } - - private class DkproPosTagger - extends EnglishPOSTagger - { - public DkproPosTagger(ObjectInputStream in) - { - super(in); - } - - @Override - protected void initMorphologicalAnalyzer() - { - mp_analyzer = new EnglishMPAnalyzer(dictModelProvider.getResource()); - } - } -} diff --git a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSegmenter.java b/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSegmenter.java deleted file mode 100644 index 2575a6baf9..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSegmenter.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.StringReader; -import java.util.List; - -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.LanguageCapability; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CasConfigurableProviderBase; -import org.dkpro.core.api.resources.ModelProviderBase; -import org.dkpro.core.api.segmentation.SegmenterBase; - -import com.clearnlp.segmentation.AbstractSegmenter; -import com.clearnlp.segmentation.EnglishSegmenter; -import com.clearnlp.tokenization.EnglishTokenizer; - -import eu.openminted.share.annotations.api.DocumentationResource; - -/** - * Tokenizer using Clear NLP. - */ -@ResourceMetaData(name = "ClearNLP Segmenter") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@LanguageCapability(value = "en") -@TypeCapability( - outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence" }) -public class ClearNlpSegmenter - extends SegmenterBase -{ - /** - * Override the default variant used to locate the model. - */ - public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; - @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) - protected String variant; - - /** - * URI of the model artifact. This can be used to override the default model resolving - * mechanism and directly address a particular model. - * - *

The URI format is {@code mvn:${groupId}:${artifactId}:${version}}. Remember to set - * the variant parameter to match the artifact. If the artifact contains the model in - * a non-default location, you also have to specify the model location parameter, e.g. - * {@code classpath:/model/path/in/artifact/model.bin}.

- */ - public static final String PARAM_MODEL_ARTIFACT_URI = - ComponentParameters.PARAM_MODEL_ARTIFACT_URI; - @ConfigurationParameter(name = PARAM_MODEL_ARTIFACT_URI, mandatory = false) - protected String modelArtifactUri; - - /** - * Load the model from this location instead of locating the model automatically. - */ - public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; - @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = false) - protected String modelLocation; - - private CasConfigurableProviderBase modelProvider; - - @Override - public void initialize(UimaContext aContext) - throws ResourceInitializationException - { - super.initialize(aContext); - - modelProvider = new ModelProviderBase(this, "segmenter") - { - { - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/segmenter-${language}-${variant}.properties"); - } - - @Override - protected AbstractSegmenter produceResource(InputStream aStream) - throws Exception - { - String lang = getAggregatedProperties().getProperty(LANGUAGE); - AbstractSegmenter segmenter; - if (lang.equals("en")) { - segmenter = new EnglishSegmenter(new EnglishTokenizer(aStream)); - } - else { - throw new ResourceInitializationException( - new Throwable("ClearNLP segmenter supports only English")); - } - return segmenter; - } - }; - } - - @Override - protected void process(JCas aJCas, String aText, int aZoneBegin) - throws AnalysisEngineProcessException - { - modelProvider.configure(aJCas.getCas()); - AbstractSegmenter segmenter = modelProvider.getResource(); - - List> sentences = segmenter - .getSentences(new BufferedReader(new StringReader(aText))); - - int sBegin = 0; - int sEnd = 0; - int tBegin = 0; - int tEnd = 0; - - for (List sentence : sentences) { - sBegin = -1; - - for (String token : sentence) { - tBegin = aText.indexOf(token, tEnd); - tEnd = tBegin + token.length(); - - if (sBegin == -1) { - sBegin = tBegin; - } - - createToken(aJCas, aZoneBegin + tBegin, aZoneBegin + tEnd); - } - sEnd = tEnd; - - createSentence(aJCas, aZoneBegin + sBegin, aZoneBegin + sEnd); - } - } -} diff --git a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabeler.java b/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabeler.java deleted file mode 100644 index 75b648343e..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabeler.java +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static java.util.Arrays.asList; -import static org.apache.commons.io.IOUtils.closeQuietly; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.apache.uima.fit.util.JCasUtil.selectCovered; -import static org.apache.uima.util.Level.INFO; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; - -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.component.JCasAnnotator_ImplBase; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.fit.util.FSCollectionFactory; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CasConfigurableProviderBase; -import org.dkpro.core.api.resources.CasConfigurableStreamProviderBase; - -import com.clearnlp.classification.model.StringModel; -import com.clearnlp.component.AbstractComponent; -import com.clearnlp.component.AbstractStatisticalComponent; -import com.clearnlp.dependency.DEPArc; -import com.clearnlp.dependency.DEPLib; -import com.clearnlp.dependency.DEPNode; -import com.clearnlp.dependency.DEPTree; -import com.clearnlp.nlp.NLPGetter; -import com.clearnlp.nlp.NLPMode; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemArg; -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemArgLink; -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemPred; -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.ROOT; -import eu.openminted.share.annotations.api.Component; -import eu.openminted.share.annotations.api.DocumentationResource; -import eu.openminted.share.annotations.api.constants.OperationType; - -/** - * ClearNLP semantic role labeller. - */ -@Component(OperationType.ANNOTATOR_OF_SEMANTIC_ROLE_LABELS) -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@ResourceMetaData(name = "ClearNLP Semantic Role Labeler") -@TypeCapability( - inputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma", - "de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency"}, - outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemPred", - "de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemArg"} - ) -public class ClearNlpSemanticRoleLabeler - extends JCasAnnotator_ImplBase -{ - /** - * Write the tag set(s) to the log when a model is loaded. - */ - public static final String PARAM_PRINT_TAGSET = ComponentParameters.PARAM_PRINT_TAGSET; - @ConfigurationParameter(name = PARAM_PRINT_TAGSET, mandatory = true, defaultValue = "false") - protected boolean printTagSet; - - /** - * Use this language instead of the document language to resolve the model. - */ - public static final String PARAM_LANGUAGE = ComponentParameters.PARAM_LANGUAGE; - @ConfigurationParameter(name = PARAM_LANGUAGE, mandatory = false) - protected String language; - - /** - * Variant of a model the model. Used to address a specific model if here are multiple models - * for one language. - */ - public static final String PARAM_VARIANT = ComponentParameters.PARAM_VARIANT; - @ConfigurationParameter(name = PARAM_VARIANT, mandatory = false) - protected String variant; - - /** - * Location from which the predicate identifier model is read. - */ - public static final String PARAM_PRED_MODEL_LOCATION = "predModelLocation"; - @ConfigurationParameter(name = PARAM_PRED_MODEL_LOCATION, mandatory = false) - protected String predModelLocation; - - /** - * Location from which the roleset classification model is read. - */ - public static final String PARAM_ROLE_MODEL_LOCATION = "roleModelLocation"; - @ConfigurationParameter(name = PARAM_ROLE_MODEL_LOCATION, mandatory = false) - protected String roleModelLocation; - - /** - * Location from which the semantic role labeling model is read. - */ - public static final String PARAM_SRL_MODEL_LOCATION = "srlModelLocation"; - @ConfigurationParameter(name = PARAM_SRL_MODEL_LOCATION, mandatory = false) - protected String srlModelLocation; - - /** - *

Normally the arguments point only to the head words of arguments in the dependency tree. - * With this option enabled, they are expanded to the text covered by the minimal and maximal - * token offsets of all descendants (or self) of the head word.

- * - *

Warning: this parameter should be used with caution! For one, if the descentants of a - * head word cover a non-continuous region of the text, this information is lost. The arguments - * will appear to be spanning a continuous region. For another, the arguments may overlap with - * each other. E.g. if a sentence contains a relative clause with a verb, the subject of the - * main clause may be recognized as a dependent of the verb and may cause the whole main - * clause to be recorded in the argument.

- */ - public static final String PARAM_EXPAND_ARGUMENTS = "expandArguments"; - @ConfigurationParameter(name = PARAM_EXPAND_ARGUMENTS, mandatory = true, defaultValue = "false") - protected boolean expandArguments; - - - private CasConfigurableProviderBase predicateFinder; - - private CasConfigurableProviderBase roleSetClassifier; - - private CasConfigurableProviderBase roleLabeller; - - @Override - public void initialize(UimaContext aContext) - throws ResourceInitializationException - { - super.initialize(aContext); - - predicateFinder = new CasConfigurableStreamProviderBase() - { - { - setContextObject(ClearNlpSemanticRoleLabeler.this); - - setDefault(GROUP_ID, "de.tudarmstadt.ukp.dkpro.core"); - setDefault(ARTIFACT_ID, "${groupId}.clearnlp-model-pred-${language}-${variant}"); - setDefault(LOCATION, - "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/pred-${language}-${variant}.properties"); - setDefault(VARIANT, "ontonotes"); - - setOverride(LOCATION, predModelLocation); - setOverride(LANGUAGE, language); - setOverride(VARIANT, variant); - } - - @Override - protected AbstractComponent produceResource(InputStream aStream) - throws Exception - { - BufferedInputStream bis = null; - ObjectInputStream ois = null; - GZIPInputStream gis = null; - try { - gis = new GZIPInputStream(aStream); - bis = new BufferedInputStream(gis); - ois = new ObjectInputStream(bis); - AbstractComponent component = NLPGetter.getComponent(ois, - getAggregatedProperties().getProperty(LANGUAGE), NLPMode.MODE_PRED); - printTags(NLPMode.MODE_PRED, component); - return component; - } - catch (Exception e) { - throw new IOException(e); - } - finally { - closeQuietly(ois); - closeQuietly(bis); - closeQuietly(gis); - } - } - }; - - roleSetClassifier = new CasConfigurableStreamProviderBase() - { - { - setContextObject(ClearNlpSemanticRoleLabeler.this); - - setDefault(ARTIFACT_ID, "${groupId}.clearnlp-model-role-${language}-${variant}"); - setDefault(LOCATION, "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/" - + "role-${language}-${variant}.properties"); - setDefault(VARIANT, "ontonotes"); - - setOverride(LOCATION, roleModelLocation); - setOverride(LANGUAGE, language); - setOverride(VARIANT, variant); - } - - @Override - protected AbstractComponent produceResource(InputStream aStream) - throws Exception - { - BufferedInputStream bis = null; - ObjectInputStream ois = null; - GZIPInputStream gis = null; - try { - gis = new GZIPInputStream(aStream); - bis = new BufferedInputStream(gis); - ois = new ObjectInputStream(bis); - AbstractComponent component = NLPGetter.getComponent(ois, - getAggregatedProperties().getProperty(LANGUAGE), NLPMode.MODE_ROLE); - - printTags(NLPMode.MODE_ROLE, component); - return component; - } - catch (Exception e) { - throw new IOException(e); - } - finally { - closeQuietly(ois); - closeQuietly(bis); - closeQuietly(gis); - } - } - }; - - roleLabeller = new CasConfigurableStreamProviderBase() - { - { - setContextObject(ClearNlpSemanticRoleLabeler.this); - - setDefault(ARTIFACT_ID, "${groupId}.clearnlp-model-srl-${language}-${variant}"); - setDefault(LOCATION, "classpath:/de/tudarmstadt/ukp/dkpro/core/clearnlp/lib/" - + "srl-${language}-${variant}.properties"); - setDefault(VARIANT, "ontonotes"); - - setOverride(LOCATION, srlModelLocation); - setOverride(LANGUAGE, language); - setOverride(VARIANT, variant); - } - - @Override - protected AbstractComponent produceResource(InputStream aStream) - throws Exception - { - BufferedInputStream bis = null; - ObjectInputStream ois = null; - GZIPInputStream gis = null; - try { - gis = new GZIPInputStream(aStream); - bis = new BufferedInputStream(gis); - ois = new ObjectInputStream(bis); - AbstractComponent component = NLPGetter.getComponent(ois, - getAggregatedProperties().getProperty(LANGUAGE), NLPMode.MODE_SRL); - printTags(NLPMode.MODE_SRL, component); - return component; - } - catch (Exception e) { - throw new IOException(e); - } - finally { - closeQuietly(ois); - closeQuietly(bis); - closeQuietly(gis); - } - } - }; - } - - @Override - public void process(JCas aJCas) - throws AnalysisEngineProcessException - { - predicateFinder.configure(aJCas.getCas()); - roleSetClassifier.configure(aJCas.getCas()); - roleLabeller.configure(aJCas.getCas()); - - // Iterate over all sentences - for (Sentence sentence : select(aJCas, Sentence.class)) { - List tokens = selectCovered(aJCas, Token.class, sentence); - DEPTree tree = new DEPTree(); - - // Generate: - // - DEPNode - // - pos tags - // - lemma - for (int i = 0; i < tokens.size(); i++) { - Token t = tokens.get(i); - DEPNode node = new DEPNode(i + 1, tokens.get(i).getText()); - node.pos = t.getPos().getPosValue(); - node.lemma = t.getLemma().getValue(); - tree.add(node); - } - - // Generate: - // Dependency relations - for (Dependency dep : selectCovered(Dependency.class, sentence)) { - if (dep instanceof ROOT) { - // #736 ClearNlpSemanticRoleLabelerTest gets caught in infinite loop - // ClearNLP parser creates roots that do not have a head. We have to replicate - // this here to avoid running into an endless loop. - continue; - } - - int headIndex = tokens.indexOf(dep.getGovernor()); - int tokenIndex = tokens.indexOf(dep.getDependent()); - - DEPNode token = tree.get(tokenIndex + 1); - DEPNode head = tree.get(headIndex + 1); - - token.setHead(head, dep.getDependencyType()); - } - - // For the root node - for (int i = 0; i < tokens.size(); i++) { - DEPNode parserNode = tree.get(i + 1); - if (parserNode.getLabel() == null) { - int headIndex = tokens.indexOf(null); - DEPNode head = tree.get(headIndex + 1); - parserNode.setHead(head, "root"); - } - } - - // Do the SRL - predicateFinder.getResource().process(tree); - roleSetClassifier.getResource().process(tree); - roleLabeller.getResource().process(tree); - - // Convert the results into UIMA annotations - Map predicates = new HashMap<>(); - Map> predArgs = new HashMap<>(); - - for (int i = 0; i < tokens.size(); i++) { - DEPNode parserNode = tree.get(i + 1); - Token argumentToken = tokens.get(i); - - for (DEPArc argPredArc : parserNode.getSHeads()) { - Token predToken = tokens.get(argPredArc.getNode().id - 1); - - // Instantiate the semantic predicate annotation if it hasn't been done yet - SemPred pred = predicates.get(predToken); - if (pred == null) { - // Create the semantic predicate annotation itself - pred = new SemPred(aJCas, predToken.getBegin(), predToken.getEnd()); - pred.setCategory(argPredArc.getNode().getFeat(DEPLib.FEAT_PB)); - pred.addToIndexes(); - predicates.put(predToken, pred); - - // Prepare a list to store its arguments - predArgs.put(pred, new ArrayList<>()); - } - - // Instantiate the semantic argument annotation - SemArg arg = new SemArg(aJCas); - - if (expandArguments) { - List descendents = parserNode.getDescendents(Integer.MAX_VALUE) - .stream() - .map(arc -> arc.getNode()) - .collect(Collectors.toList()); - descendents.add(parserNode); - List descTokens = descendents.stream() - .map(node -> tokens.get(node.id - 1)) - .collect(Collectors.toList()); - int begin = descTokens.stream().mapToInt(t -> t.getBegin()).min() - .getAsInt(); - int end = descTokens.stream().mapToInt(t -> t.getEnd()).max().getAsInt(); - arg.setBegin(begin); - arg.setEnd(end); - } - else { - arg.setBegin(argumentToken.getBegin()); - arg.setEnd(argumentToken.getEnd()); - } - - arg.addToIndexes(); - - SemArgLink link = new SemArgLink(aJCas); - link.setRole(argPredArc.getLabel()); - link.setTarget(arg); - - // Remember to which predicate this argument belongs - predArgs.get(pred).add(link); - } - } - - for (Entry> e : predArgs.entrySet()) { - e.getKey().setArguments(FSCollectionFactory.createFSArray(aJCas, e.getValue())); - } - } - } - - private void printTags(String aType, AbstractComponent aComponent) - { - if (printTagSet && (aComponent instanceof AbstractStatisticalComponent)) { - AbstractStatisticalComponent component = (AbstractStatisticalComponent) aComponent; - - Set tagSet = new HashSet(); - - for (StringModel model : component.getModels()) { - tagSet.addAll(asList(model.getLabels())); - } - - List tagList = new ArrayList(tagSet); - Collections.sort(tagList); - - StringBuilder sb = new StringBuilder(); - sb.append("Model of " + aType + " contains [").append(tagList.size()) - .append("] tags: "); - - for (String tag : tagList) { - sb.append(tag); - sb.append(" "); - } - getContext().getLogger().log(INFO, sb.toString()); - } - } -} diff --git a/dkpro-core-clearnlp-asl/src/main/resources/org/dkpro/core/clearnlp/lib/parser-default-variants.map b/dkpro-core-clearnlp-asl/src/main/resources/org/dkpro/core/clearnlp/lib/parser-default-variants.map deleted file mode 100644 index 8c7589c3c4..0000000000 --- a/dkpro-core-clearnlp-asl/src/main/resources/org/dkpro/core/clearnlp/lib/parser-default-variants.map +++ /dev/null @@ -1 +0,0 @@ -en=ontonotes diff --git a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpLemmatizerTest.java b/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpLemmatizerTest.java deleted file mode 100644 index 0a003c4abe..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpLemmatizerTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.util.JCasUtil.select; - -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.testing.AssertAnnotations; -import org.dkpro.core.testing.TestRunner; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma; - -public class ClearNlpLemmatizerTest -{ - @Test - public void testEnglish() - throws Exception - { - // Assume.assumeTrue(Runtime.getRuntime().maxMemory() > 1200000000l); - - JCas jcas = runTest("en", "We need a very complicated example sentence , which " - + "contains as many constituents and dependencies as possible ."); - - String[] lemmas = { "we", "need", "a", "very", "complicated", "example", "sentence", ",", - "which", "contain", "as", "many", "constituent", "and", "dependency", "as", - "possible", "." }; - - AssertAnnotations.assertLemma(lemmas, select(jcas, Lemma.class)); - } - - @Test - public void testUnderscore() - throws Exception - { - JCas jcas = runTest("en", "foo _ bar"); - - String[] lemmas = { "foo", "_", "bar" }; - - AssertAnnotations.assertLemma(lemmas, select(jcas, Lemma.class)); - } - - private JCas runTest(String aLanguage, String aText) - throws Exception - { - AnalysisEngineDescription tagger = createEngineDescription(ClearNlpPosTagger.class); - AnalysisEngineDescription lemma = createEngineDescription(ClearNlpLemmatizer.class); - - JCas jcas = TestRunner.runTest(createEngineDescription(tagger, lemma), aLanguage, aText); - - return jcas; - } -} diff --git a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpParserTest.java b/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpParserTest.java deleted file mode 100644 index 5bf2695591..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpParserTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.opennlp.OpenNlpPosTagger; -import org.dkpro.core.testing.AssertAnnotations; -import org.dkpro.core.testing.TestRunner; -import org.dkpro.core.testing.dumper.DependencyDumper; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; - -public class ClearNlpParserTest -{ - static final String documentEnglish = "We need a very complicated example sentence , which " + - "contains as many constituents and dependencies as possible ."; - - @Test - public void testEnglishDependencies() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 3000000000l); - - JCas jcas = runTest("en", null, documentEnglish); - - String[] dependencies = new String[] { - "[ 0, 2]Dependency(nsubj,basic) D[0,2](We) G[3,7](need)", - "[ 3, 7]ROOT(ROOT,basic) D[3,7](need) G[3,7](need)", - "[ 8, 9]Dependency(det,basic) D[8,9](a) G[35,43](sentence)", - "[ 10, 14]Dependency(advmod,basic) D[10,14](very) G[15,26](complicated)", - "[ 15, 26]Dependency(amod,basic) D[15,26](complicated) G[35,43](sentence)", - "[ 27, 34]Dependency(nn,basic) D[27,34](example) G[35,43](sentence)", - "[ 35, 43]Dependency(dobj,basic) D[35,43](sentence) G[3,7](need)", - "[ 44, 45]Dependency(punct,basic) D[44,45](,) G[35,43](sentence)", - "[ 46, 51]Dependency(nsubj,basic) D[46,51](which) G[52,60](contains)", - "[ 52, 60]Dependency(rcmod,basic) D[52,60](contains) G[35,43](sentence)", - "[ 61, 63]Dependency(prep,basic) D[61,63](as) G[52,60](contains)", - "[ 64, 68]Dependency(amod,basic) D[64,68](many) G[69,81](constituents)", - "[ 69, 81]Dependency(pobj,basic) D[69,81](constituents) G[61,63](as)", - "[ 82, 85]Dependency(cc,basic) D[82,85](and) G[69,81](constituents)", - "[ 86, 98]Dependency(conj,basic) D[86,98](dependencies) G[69,81](constituents)", - "[ 99,101]Dependency(prep,basic) D[99,101](as) G[86,98](dependencies)", - "[102,110]Dependency(amod,basic) D[102,110](possible) G[99,101](as)", - "[111,112]Dependency(punct,basic) D[111,112](.) G[3,7](need)" }; - - AssertAnnotations.assertDependencies(dependencies, select(jcas, Dependency.class)); - } - - @Test - public void testEnglishMayo() - throws Exception - { -// Assume.assumeTrue(Runtime.getRuntime().maxMemory() > 1200000000l); - - JCas jcas = runTest("en", "mayo", documentEnglish); - - String[] dependencies = new String[] { - "[ 0, 2]Dependency(nsubj,basic) D[0,2](We) G[3,7](need)", - "[ 3, 7]ROOT(ROOT,basic) D[3,7](need) G[3,7](need)", - "[ 8, 9]Dependency(det,basic) D[8,9](a) G[35,43](sentence)", - "[ 10, 14]Dependency(advmod,basic) D[10,14](very) G[15,26](complicated)", - "[ 15, 26]Dependency(amod,basic) D[15,26](complicated) G[35,43](sentence)", - "[ 27, 34]Dependency(nn,basic) D[27,34](example) G[35,43](sentence)", - "[ 35, 43]Dependency(dobj,basic) D[35,43](sentence) G[3,7](need)", - "[ 44, 45]Dependency(punct,basic) D[44,45](,) G[35,43](sentence)", - "[ 46, 51]Dependency(nsubj,basic) D[46,51](which) G[52,60](contains)", - "[ 52, 60]Dependency(rcmod,basic) D[52,60](contains) G[35,43](sentence)", - "[ 61, 63]Dependency(prep,basic) D[61,63](as) G[52,60](contains)", - "[ 64, 68]Dependency(amod,basic) D[64,68](many) G[69,81](constituents)", - "[ 69, 81]Dependency(pobj,basic) D[69,81](constituents) G[61,63](as)", - "[ 82, 85]Dependency(cc,basic) D[82,85](and) G[69,81](constituents)", - "[ 86, 98]Dependency(conj,basic) D[86,98](dependencies) G[69,81](constituents)", - "[ 99,101]Dependency(mark,basic) D[99,101](as) G[102,110](possible)", - "[102,110]Dependency(advcl,basic) D[102,110](possible) G[52,60](contains)", - "[111,112]Dependency(punct,basic) D[111,112](.) G[3,7](need)" }; - - AssertAnnotations.assertDependencies(dependencies, select(jcas, Dependency.class)); - } - - private JCas runTest(String aLanguage, String aVariant, String aText) - throws Exception - { - AnalysisEngineDescription engine = createEngineDescription( - createEngineDescription(OpenNlpPosTagger.class), - createEngineDescription(ClearNlpLemmatizer.class), - createEngineDescription(ClearNlpParser.class, - ClearNlpParser.PARAM_VARIANT, aVariant, - ClearNlpParser.PARAM_PRINT_TAGSET, true), - createEngineDescription(DependencyDumper.class)); - - return TestRunner.runTest(engine, aLanguage, aText); - } -} diff --git a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpPosTaggerTest.java b/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpPosTaggerTest.java deleted file mode 100644 index 5c5168797b..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpPosTaggerTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.apache.uima.analysis_engine.AnalysisEngine; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.testing.AssertAnnotations; -import org.dkpro.core.testing.TestRunner; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; - -public class ClearNlpPosTaggerTest -{ - @Test - public void testEnglish() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 1200000000l); - - runTest("en", null, "This is a test . \n", - new String[] { "DT", "VBZ", "DT", "NN", "." }, - new String[] { "POS_DET", "POS_VERB", "POS_DET", "POS_NOUN", "POS_PUNCT" }); - - runTest("en", null, "A neural net . \n", - new String[] { "DT", "JJ", "NN", "." }, - new String[] { "POS_DET", "POS_ADJ", "POS_NOUN", "POS_PUNCT" }); - - runTest("en", null, "John is purchasing oranges . \n", - new String[] { "NNP", "VBZ", "VBG", "NNS", "." }, - new String[] { "POS_PROPN", "POS_VERB", "POS_VERB", "POS_NOUN", "POS_PUNCT" }); - } - - @Test - public void testEnglishMayo() - throws Exception - { - runTest("en", "mayo", "This is a test . \n", - new String[] { "DT", "VBZ", "DT", "NN", "." }, - new String[] { "POS_DET", "POS_VERB", "POS_DET", "POS_NOUN", "POS_PUNCT" }); - - runTest("en", "mayo", "A neural net . \n", - new String[] { "DT", "JJ", "NN", "." }, - new String[] { "POS_DET", "POS_ADJ", "POS_NOUN", "POS_PUNCT" }); - - runTest("en", "mayo", "John is purchasing oranges . \n", - new String[] { "NNP", "VBZ", "VBG", "NNS", "." }, - new String[] { "POS_PROPN", "POS_VERB", "POS_VERB", "POS_NOUN", "POS_PUNCT" }); - } - - private void runTest(String language, String variant, String testDocument, String[] tags, - String[] tagClasses) - throws Exception - { - AnalysisEngine engine = createEngine(ClearNlpPosTagger.class, - ClearNlpPosTagger.PARAM_VARIANT, variant, - ClearNlpPosTagger.PARAM_PRINT_TAGSET, true); - - JCas jcas = TestRunner.runTest(engine, language, testDocument); - - AssertAnnotations.assertPOS(tagClasses, tags, select(jcas, POS.class)); - } - - @BeforeEach - public void clearMemory() - { - Runtime.getRuntime().gc(); - Runtime.getRuntime().gc(); - Runtime.getRuntime().gc(); - } -} diff --git a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSegmenterTest.java b/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSegmenterTest.java deleted file mode 100644 index f15d3e1d64..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSegmenterTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine; -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.uima.analysis_engine.AnalysisEngine; -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.testing.harness.SegmenterHarness; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; - -public class ClearNlpSegmenterTest -{ - @Test - public void run() throws Throwable - { - AnalysisEngineDescription aed = createEngineDescription(ClearNlpSegmenter.class); - SegmenterHarness.run(aed, "de.1", "de.2", "de.3", "de.4", "en.1", "en.7", "en.9", "ar.1", - "zh.1", "zh.2"); - } - - /** - * We had a bug where the token offsets were assigned wrong when one word was a suffix of the - * previous word. - */ - @Test - public void testSuffix() throws Exception - { - JCas jcas = JCasFactory.createJCas(); - jcas.setDocumentLanguage("en"); - jcas.setDocumentText("this is is this is is"); - - AnalysisEngine aed = createEngine(ClearNlpSegmenter.class); - aed.process(jcas); - - - List tokens = new ArrayList<>(select(jcas, Token.class)); - assertEquals(5, tokens.get(1).getBegin()); - assertEquals(7, tokens.get(1).getEnd()); - - for (Token t : tokens) { - System.out.printf("%d %d %s%n", t.getBegin(), t.getEnd(), t.getCoveredText()); - } - - } - - @Test - public void testZoning() throws Exception - { - SegmenterHarness.testZoning(ClearNlpSegmenter.class); - } -} diff --git a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabelerTest.java b/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabelerTest.java deleted file mode 100644 index 0dc8513d3f..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/java/org/dkpro/core/clearnlp/ClearNlpSemanticRoleLabelerTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.clearnlp; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.apache.commons.lang3.ArrayUtils; -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.opennlp.OpenNlpPosTagger; -import org.dkpro.core.testing.AssertAnnotations; -import org.dkpro.core.testing.TestRunner; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemPred; - -public class ClearNlpSemanticRoleLabelerTest -{ - static final String documentEnglish = "We need a very complicated example sentence , which " - + "contains as many constituents and dependencies as possible ."; - - @Test - public void testEnglish() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 3000000000l); - - JCas jcas = runTest("en", null, documentEnglish); - - String[] predicates = { - "contains (contain.01): [(A0:sentence)(A1:as)(R-A0:which)]", - "need (need.01): [(A0:We)(A1:sentence)]" }; - - AssertAnnotations.assertSemPred(predicates, select(jcas, SemPred.class)); - } - - @Test - public void testEnglishExpand() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 3000000000l); - - JCas jcas = runTest("en", null, documentEnglish, - ClearNlpSemanticRoleLabeler.PARAM_EXPAND_ARGUMENTS, true); - - String[] predicates = { - "contains (contain.01): [" - + "(A0:a very complicated example sentence , which contains as many constituents and dependencies as possible)" - + "(A1:as many constituents and dependencies as possible)" - + "(R-A0:which)]", - "need (need.01): [" - + "(A0:We)" - + "(A1:a very complicated example sentence , which contains as many constituents and dependencies as possible)]" - }; - - AssertAnnotations.assertSemPred(predicates, select(jcas, SemPred.class)); - } - - @Test - public void testEnglishExpand2() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 3000000000l); - - JCas jcas = runTest("en", null, "The man was sued by Jacqueline Kennedy Onassis .", - ClearNlpSemanticRoleLabeler.PARAM_EXPAND_ARGUMENTS, true); - - String[] predicates = { "sued (sue.01): [(A0:by Jacqueline Kennedy Onassis)(A1:The man)]" }; - - AssertAnnotations.assertSemPred(predicates, select(jcas, SemPred.class)); - } - @Test - public void testEnglishMayo() - throws Exception - { - assumeTrue(Runtime.getRuntime().maxMemory() > 3000000000l); - - JCas jcas = runTest("en", "mayo", documentEnglish); - - String[] predicates = { - "contains (contain.01): [(A0:sentence)(A1:as)(R-A0:which)]", - "need (need.01): [(A0:We)(A1:sentence)]" }; - - AssertAnnotations.assertSemPred(predicates, select(jcas, SemPred.class)); - } - - private JCas runTest(String aLanguage, String aVariant, String aText, Object... aExtraParams) - throws Exception - { - Object[] params = new Object[] { - ClearNlpParser.PARAM_VARIANT, aVariant, - ClearNlpParser.PARAM_PRINT_TAGSET, true}; - params = ArrayUtils.addAll(params, aExtraParams); - - AnalysisEngineDescription engine = createEngineDescription( - createEngineDescription(OpenNlpPosTagger.class), - createEngineDescription(ClearNlpLemmatizer.class), - createEngineDescription(ClearNlpParser.class), - createEngineDescription(ClearNlpSemanticRoleLabeler.class, params)); - - return TestRunner.runTest(engine, aLanguage, aText); - } - - - @BeforeEach - public void freeMemory() - { - Runtime.getRuntime().gc(); - Runtime.getRuntime().gc(); - Runtime.getRuntime().gc(); - } -} diff --git a/dkpro-core-clearnlp-asl/src/test/resources/log4j2.xml b/dkpro-core-clearnlp-asl/src/test/resources/log4j2.xml deleted file mode 100644 index 19bf03b585..0000000000 --- a/dkpro-core-clearnlp-asl/src/test/resources/log4j2.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/dkpro-core-io-web1t-asl/pom.xml b/dkpro-core-io-web1t-asl/pom.xml index a756a0776f..96fdd62392 100644 --- a/dkpro-core-io-web1t-asl/pom.xml +++ b/dkpro-core-io-web1t-asl/pom.xml @@ -122,36 +122,18 @@ de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.opennlp-model-tagger-en-maxent - test - - - org.dkpro.core - dkpro-core-clearnlp-asl - ${project.version} + de.tudarmstadt.ukp.dkpro.core.opennlp-model-tagger-de-maxent test de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-lemma-en-default - test - - - de.tudarmstadt.ukp.dkpro.core - de.tudarmstadt.ukp.dkpro.core.clearnlp-model-dictionary-en-default + de.tudarmstadt.ukp.dkpro.core.opennlp-model-lemma-de-gsd test - - org.dkpro.core - dkpro-core-clearnlp-asl - ${project.version} - pom - import - org.dkpro.core dkpro-core-opennlp-asl @@ -173,9 +155,8 @@ - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.opennlp-model-tagger-en-maxent - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-dictionary-en-default - de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.clearnlp-model-lemma-en-default + de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.opennlp-model-pos-de-maxent + de.tudarmstadt.ukp.dkpro.core:de.tudarmstadt.ukp.dkpro.core.opennlp-model-lemma-de-gsd diff --git a/dkpro-core-io-web1t-asl/src/test/java/org/dkpro/core/io/web1t/Web1TWriterTest.java b/dkpro-core-io-web1t-asl/src/test/java/org/dkpro/core/io/web1t/Web1TWriterTest.java index 733b0fd8ec..dcb509a2eb 100644 --- a/dkpro-core-io-web1t-asl/src/test/java/org/dkpro/core/io/web1t/Web1TWriterTest.java +++ b/dkpro-core-io-web1t-asl/src/test/java/org/dkpro/core/io/web1t/Web1TWriterTest.java @@ -1,14 +1,14 @@ /* - * Copyright 2011 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader; +import static org.apache.uima.fit.pipeline.SimplePipeline.runPipeline; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -26,13 +27,10 @@ import java.io.IOException; import org.apache.uima.UIMAException; -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.collection.CollectionReader; -import org.apache.uima.fit.pipeline.SimplePipeline; import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.clearnlp.ClearNlpLemmatizer; import org.dkpro.core.frequency.Web1TFileAccessProvider; import org.dkpro.core.io.text.TextReader; +import org.dkpro.core.opennlp.OpenNlpLemmatizer; import org.dkpro.core.opennlp.OpenNlpPosTagger; import org.dkpro.core.tokit.BreakIteratorSegmenter; import org.junit.jupiter.api.Test; @@ -42,84 +40,77 @@ public class Web1TWriterTest { - private final int MIN_NGRAM = 1; - private final int MAX_NGRAM = 3; + private static final int MIN_NGRAM = 1; + private static final int MAX_NGRAM = 3; @Test public void web1TFormatTestWithTwoMultiSlashedTypesAsFeaturePath(@TempDir File folder) throws Exception { - Web1TFileAccessProvider web1tProvider = prepareWeb1TFormatTest(folder, new String[] { - Token.class.getName() + "/pos/PosValue", Token.class.getName() + "/lemma/value" }); + var web1tProvider = prepareWeb1TFormatTest(folder, Token.class.getName() + "/pos/PosValue", + Token.class.getName() + "/lemma/value"); - assertEquals(1, web1tProvider.getFrequency("TO")); // "to" - assertEquals(1, web1tProvider.getFrequency("NNS")); // "sentences" - assertEquals(1, web1tProvider.getFrequency("EX")); // "there" - - assertEquals(1, web1tProvider.getFrequency("write")); - assertEquals(0, web1tProvider.getFrequency("written")); + assertEquals(4, web1tProvider.getFrequency("ART")); + assertEquals(4, web1tProvider.getFrequency("ADJA")); + assertEquals(1, web1tProvider.getFrequency("APPR")); + assertEquals(2, web1tProvider.getFrequency("testsatz")); + assertEquals(-1, web1tProvider.getFrequency("sätze")); } @Test public void web1TFormatTestWithMultiSlashedTypesAsFeaturePath(@TempDir File folder) throws Exception { - Web1TFileAccessProvider web1tProvider = prepareWeb1TFormatTest(folder, - new String[] { Token.class.getName() + "/lemma/value" }); - - assertEquals(1, web1tProvider.getFrequency("write")); - assertEquals(0, web1tProvider.getFrequency("written")); - assertEquals(4, web1tProvider.getFrequency("sentence")); + var web1tProvider = prepareWeb1TFormatTest(folder, Token.class.getName() + "/lemma/value"); + assertEquals(2, web1tProvider.getFrequency("testsatz")); + assertEquals(-1, web1tProvider.getFrequency("sätze")); + assertEquals(2, web1tProvider.getFrequency("satz")); } @Test public void web1TFormatTest_randomFrequencies(@TempDir File folder) throws Exception { - Web1TFileAccessProvider web1tProvider = prepareWeb1TFormatTest(folder, - new String[] { Token.class.getName() }); + var web1tProvider = prepareWeb1TFormatTest(folder, Token.class.getName()); assertEquals(4, web1tProvider.getFrequency(".")); - assertEquals(1, web1tProvider.getFrequency(",")); - assertEquals(3, web1tProvider.getFrequency("sentence")); - assertEquals(1, web1tProvider.getFrequency("written")); - + assertEquals(1, web1tProvider.getFrequency("Satz")); + assertEquals(2, web1tProvider.getFrequency("Testsatz")); + assertEquals(1, web1tProvider.getFrequency("geschrieben")); } @Test public void web1TFormatTest_exceptionForInvalidMinFrequency1(@TempDir File folder) throws Exception { - assertThatExceptionOfType(ResourceInitializationException.class).isThrownBy( - () -> writeWeb1TFormat(folder, new String[] { Token.class.getName() }, -1)); - + assertThatExceptionOfType(ResourceInitializationException.class) + .isThrownBy(() -> writeWeb1TFormat(folder, -1, Token.class.getName())); } @Test public void web1TFormatTest_exceptionForInvalidMinFrequency2(@TempDir File folder) throws Exception { - assertThatExceptionOfType(ResourceInitializationException.class).isThrownBy( - () -> writeWeb1TFormat(folder, new String[] { Token.class.getName() }, 0)); - + assertThatExceptionOfType(ResourceInitializationException.class) + .isThrownBy(() -> writeWeb1TFormat(folder, 0, Token.class.getName())); } - private void writeWeb1TFormat(File aFolder, String[] strings, int minFreq) + private void writeWeb1TFormat(File aFolder, int minFreq, String... strings) throws UIMAException, IOException { - CollectionReader reader = createReader(TextReader.class, // - TextReader.PARAM_LANGUAGE, "en", // - TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/", // - TextReader.PARAM_PATTERNS, new String[] { "[+]**/*.txt" }); + var reader = createReader(TextReader.class, // + TextReader.PARAM_LANGUAGE, "de", // + TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/data", // + TextReader.PARAM_PATTERNS, "**/*.txt"); - AnalysisEngineDescription segmenter = createEngineDescription(BreakIteratorSegmenter.class); + var segmenter = createEngineDescription(BreakIteratorSegmenter.class); - AnalysisEngineDescription tagger = createEngineDescription(OpenNlpPosTagger.class); + var tagger = createEngineDescription(OpenNlpPosTagger.class); - AnalysisEngineDescription lemmatizer = createEngineDescription(ClearNlpLemmatizer.class); + var lemmatizer = createEngineDescription(OpenNlpLemmatizer.class); - AnalysisEngineDescription ngramWriter = createEngineDescription( // + var ngramWriter = createEngineDescription( // Web1TWriter.class, // Web1TWriter.PARAM_TARGET_LOCATION, aFolder, // Web1TWriter.PARAM_INPUT_TYPES, strings, // @@ -127,41 +118,38 @@ private void writeWeb1TFormat(File aFolder, String[] strings, int minFreq) Web1TWriter.PARAM_MAX_NGRAM_LENGTH, MAX_NGRAM, // Web1TWriter.PARAM_MIN_FREQUENCY, minFreq); - SimplePipeline.runPipeline(reader, segmenter, tagger, lemmatizer, ngramWriter); + runPipeline(reader, segmenter, tagger, lemmatizer, ngramWriter); } - private Web1TFileAccessProvider prepareWeb1TFormatTest(File target, String[] inputTypes) + private Web1TFileAccessProvider prepareWeb1TFormatTest(File target, String... inputTypes) throws Exception { writeWeb1TFormat(target, inputTypes); - Web1TFileAccessProvider web1tProvider = new Web1TFileAccessProvider("en", target, MIN_NGRAM, - MAX_NGRAM); - - return web1tProvider; + return new Web1TFileAccessProvider("de", target, MIN_NGRAM, MAX_NGRAM); } - private void writeWeb1TFormat(File target, String[] inputPath) throws Exception + private void writeWeb1TFormat(File target, String... inputPath) throws Exception { - CollectionReader reader = createReader( // + var reader = createReader( // TextReader.class, // - TextReader.PARAM_LANGUAGE, "en", // - TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/", // - TextReader.PARAM_PATTERNS, new String[] { "[+]**/*.txt" }); + TextReader.PARAM_LANGUAGE, "de", // + TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/data", // + TextReader.PARAM_PATTERNS, "**/*.txt"); - AnalysisEngineDescription segmenter = createEngineDescription(BreakIteratorSegmenter.class); + var segmenter = createEngineDescription(BreakIteratorSegmenter.class); - AnalysisEngineDescription tagger = createEngineDescription(OpenNlpPosTagger.class); + var tagger = createEngineDescription(OpenNlpPosTagger.class); - AnalysisEngineDescription lemmatizer = createEngineDescription(ClearNlpLemmatizer.class); + var lemmatizer = createEngineDescription(OpenNlpLemmatizer.class); - AnalysisEngineDescription ngramWriter = createEngineDescription( // + var ngramWriter = createEngineDescription( // Web1TWriter.class, // Web1TWriter.PARAM_TARGET_LOCATION, target, // Web1TWriter.PARAM_INPUT_TYPES, inputPath, // Web1TWriter.PARAM_MIN_NGRAM_LENGTH, MIN_NGRAM, // Web1TWriter.PARAM_MAX_NGRAM_LENGTH, MAX_NGRAM); - SimplePipeline.runPipeline(reader, segmenter, tagger, lemmatizer, ngramWriter); + runPipeline(reader, segmenter, tagger, lemmatizer, ngramWriter); } } diff --git a/dkpro-core-io-web1t-asl/src/test/resources/data/test1.txt b/dkpro-core-io-web1t-asl/src/test/resources/data/test1.txt new file mode 100644 index 0000000000..d4c02ce87b --- /dev/null +++ b/dkpro-core-io-web1t-asl/src/test/resources/data/test1.txt @@ -0,0 +1 @@ +Dies ist ein Testsatz. Das ist ein weiterer Satz. \ No newline at end of file diff --git a/dkpro-core-io-web1t-asl/src/test/resources/data/test2.txt b/dkpro-core-io-web1t-asl/src/test/resources/data/test2.txt new file mode 100644 index 0000000000..db83362415 --- /dev/null +++ b/dkpro-core-io-web1t-asl/src/test/resources/data/test2.txt @@ -0,0 +1 @@ +In der zweiten Testdatei gibt es einen weiteren Testsatz. Es müssen keine weiteren Sätze geschrieben werden. \ No newline at end of file diff --git a/dkpro-core-io-web1t-asl/src/test/resources/log4j2.xml b/dkpro-core-io-web1t-asl/src/test/resources/log4j2-test.xml similarity index 100% rename from dkpro-core-io-web1t-asl/src/test/resources/log4j2.xml rename to dkpro-core-io-web1t-asl/src/test/resources/log4j2-test.xml diff --git a/dkpro-core-io-web1t-asl/src/test/resources/test1.txt b/dkpro-core-io-web1t-asl/src/test/resources/test1.txt deleted file mode 100644 index 047d8d7fcc..0000000000 --- a/dkpro-core-io-web1t-asl/src/test/resources/test1.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test sentence. This is another sentence. \ No newline at end of file diff --git a/dkpro-core-io-web1t-asl/src/test/resources/test2.txt b/dkpro-core-io-web1t-asl/src/test/resources/test2.txt deleted file mode 100644 index ca4f4be787..0000000000 --- a/dkpro-core-io-web1t-asl/src/test/resources/test2.txt +++ /dev/null @@ -1 +0,0 @@ -In the second test file, there is another test sentence. No more sentences need to be written. \ No newline at end of file