Merge pull request #16 from diging/develop

prepare release

cepheus/pom.xml

@@ -14,6 +14,8 @@
         <geco.requests.version>0.6</geco.requests.version>
         <geco.util.version>0.4.2</geco.util.version>
         <geco.cepheus.version>0.2.2-SNAPSHOT</geco.cepheus.version>
+        <geco.kafka-util.version>0.1</geco.kafka-util.version>
+        <geco.september.util.version>0.2</geco.september.util.version>
 
         <admin.password>admin</admin.password>
         <pullrequest></pullrequest>
@@ -54,6 +56,18 @@
             <artifactId>util</artifactId>
             <version>${geco.util.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>edu.asu.diging.giles-eco</groupId>
+            <artifactId>kafka-util</artifactId>
+            <version>${geco.kafka-util.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>edu.asu.diging.giles-eco</groupId>
+            <artifactId>september-util</artifactId>
+            <version>${geco.september.util.version}</version>
+        </dependency>
 
         <!-- Spring -->
         <dependency>

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/config/CepheusConfig.java

@@ -0,0 +1,22 @@
+package edu.asu.diging.gilesecosystem.cepheus.config;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import edu.asu.diging.gilesecosystem.cepheus.service.Properties;
+import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler;
+import edu.asu.diging.gilesecosystem.septemberutil.service.impl.SystemMessageHandler;
+import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;
+
+@Configuration
+public class CepheusConfig {
+
+    @Autowired
+    private IPropertiesManager propertyManager;
+
+    @Bean
+    public ISystemMessageHandler getMessageHandler() {
+        return new SystemMessageHandler(propertyManager.getProperty(Properties.APPLICATION_ID));
+    }
+}

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/config/CepheusKafkaConfig.java

@@ -16,8 +16,9 @@
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 
 import edu.asu.diging.gilesecosystem.cepheus.kafka.ExtractionRequestReceiver;
-import edu.asu.diging.gilesecosystem.cepheus.service.IPropertiesManager;
+import edu.asu.diging.gilesecosystem.cepheus.service.Properties;
 import edu.asu.diging.gilesecosystem.requests.kafka.KafkaConfig;
+import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;
 
 @Configuration
 @EnableKafka
@@ -65,7 +66,7 @@ public ExtractionRequestReceiver receiver() {
 
     @Override
     public String getHosts() {
-        return propertiesManager.getProperty(IPropertiesManager.KAFKA_HOSTS);
+        return propertiesManager.getProperty(Properties.KAFKA_HOSTS);
     }
 
     @Override

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/kafka/ExtractionRequestReceiver.java

@@ -13,7 +13,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import edu.asu.diging.gilesecosystem.cepheus.exceptions.CepheusExtractionException;
-import edu.asu.diging.gilesecosystem.cepheus.service.IPropertiesManager;
 import edu.asu.diging.gilesecosystem.cepheus.service.pdf.IImageExtractionManager;
 import edu.asu.diging.gilesecosystem.requests.IImageExtractionRequest;
 import edu.asu.diging.gilesecosystem.requests.impl.ImageExtractionRequest;
@@ -26,10 +25,6 @@ public class ExtractionRequestReceiver {
     @Autowired
     private IImageExtractionManager imageExtractionManager;
 
-    @Autowired
-    protected IPropertiesManager propertiesManager;
-
-
     @KafkaListener(id="cepheus.extraction", topics = {"${topic_extract_images_request}"})
     public void receiveMessage(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String topic) {
         extractImage(message);

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/service/IPropertiesManager.java → cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/service/Properties.java

@@ -1,12 +1,8 @@
 package edu.asu.diging.gilesecosystem.cepheus.service;
 
-import java.util.Map;
-
-import edu.asu.diging.gilesecosystem.cepheus.exceptions.CepheusPropertiesStorageException;
-
-public interface IPropertiesManager {
+public interface Properties {
 
-    public final static String CEPHEUS_URL = "cepheus_url";
+    public final static String CEPHEUS_URL = "app_url";
     public final static String PDF_TO_IMAGE_DPI = "pdf_to_image_dpi";
     public final static String PDF_TO_IMAGE_TYPE = "pdf_to_image_type";
     public final static String PDF_TO_IMAGE_FORMAT = "pdf_to_image_format";
@@ -15,13 +11,7 @@ public interface IPropertiesManager {
     public final static String KAFKA_IMAGE_EXTRACTION_TOPIC = "topic_extract_images_request";
     public final static String KAFKA_IMAGE_EXTRACTION_COMPLETE_TOPIC = "topic_extract_images_request_complete";
     public final static String GILES_ACCESS_TOKEN = "giles_access_token";
-
-
-    public abstract void setProperty(String key, String value) throws CepheusPropertiesStorageException;
-
-    public abstract String getProperty(String key);
+    public final static String APPLICATION_ID = "application_id";
 
-    public abstract void updateProperties(Map<String, String> props)
-            throws CepheusPropertiesStorageException;
 
 }

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/service/pdf/impl/AExtractionManager.java

@@ -18,8 +18,9 @@
 import org.springframework.http.converter.ByteArrayHttpMessageConverter;
 import org.springframework.web.client.RestTemplate;
 
-import edu.asu.diging.gilesecosystem.cepheus.service.IPropertiesManager;
+import edu.asu.diging.gilesecosystem.cepheus.service.Properties;
 import edu.asu.diging.gilesecosystem.util.files.IFileStorageManager;
+import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;
 
 public class AExtractionManager {
 
@@ -89,7 +90,7 @@ protected Page saveTextToFile(int pageNr, String requestId,
     }
 
     protected String getRestEndpoint() {
-        String restEndpoint = propertiesManager.getProperty(IPropertiesManager.CEPHEUS_URL);
+        String restEndpoint = propertiesManager.getProperty(Properties.CEPHEUS_URL);
         if (restEndpoint.endsWith("/")) {
             restEndpoint = restEndpoint.substring(0, restEndpoint.length()-1);
         }

cepheus/src/main/java/edu/asu/diging/gilesecosystem/cepheus/service/pdf/impl/ImageExtractionManager.java

@@ -26,7 +26,7 @@
 
 import edu.asu.diging.gilesecosystem.cepheus.exceptions.CepheusExtractionException;
 import edu.asu.diging.gilesecosystem.cepheus.rest.DownloadFileController;
-import edu.asu.diging.gilesecosystem.cepheus.service.IPropertiesManager;
+import edu.asu.diging.gilesecosystem.cepheus.service.Properties;
 import edu.asu.diging.gilesecosystem.cepheus.service.pdf.IImageExtractionManager;
 import edu.asu.diging.gilesecosystem.requests.ICompletedImageExtractionRequest;
 import edu.asu.diging.gilesecosystem.requests.IImageExtractionRequest;
@@ -37,6 +37,7 @@
 import edu.asu.diging.gilesecosystem.requests.impl.CompletedImageExtractionRequest;
 import edu.asu.diging.gilesecosystem.requests.kafka.IRequestProducer;
 import edu.asu.diging.gilesecosystem.util.files.IFileStorageManager;
+import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;
 
 @Service
 public class ImageExtractionManager extends AExtractionManager implements
@@ -83,65 +84,70 @@ public void extractImages(IImageExtractionRequest request)
             throws CepheusExtractionException {
         logger.info("Extracting images for: " + request.getDownloadUrl());
 
-        String dpi = propertiesManager.getProperty(IPropertiesManager.PDF_TO_IMAGE_DPI)
+        String dpi = propertiesManager.getProperty(Properties.PDF_TO_IMAGE_DPI)
                 .trim();
-        String type = propertiesManager.getProperty(IPropertiesManager.PDF_TO_IMAGE_TYPE)
+        String type = propertiesManager.getProperty(Properties.PDF_TO_IMAGE_TYPE)
                 .trim();
         String format = propertiesManager.getProperty(
-                IPropertiesManager.PDF_TO_IMAGE_FORMAT).trim();
+                Properties.PDF_TO_IMAGE_FORMAT).trim();
 
-        PDDocument pdfDocument;
+        PDDocument pdfDocument = null;
+        RequestStatus status = RequestStatus.COMPLETE;
         try {
             pdfDocument = PDDocument.load(new ByteArrayInputStream(downloadFile(request.getDownloadUrl())), MemoryUsageSetting.setupTempFileOnly());
         } catch (IOException e) {
-            throw new CepheusExtractionException(e);
+            logger.error("Could not extract text.", e);
+            status = RequestStatus.FAILED;
         }
 
-        int numPages = pdfDocument.getNumberOfPages();
-        PDFRenderer renderer = new PDFRenderer(pdfDocument);
         List<edu.asu.diging.gilesecosystem.requests.impl.Page> pages = new ArrayList<>();
-
-        String restEndpoint = getRestEndpoint();
-
-        for (int i = 0; i < numPages; i++) {
-            edu.asu.diging.gilesecosystem.requests.impl.Page requestPage = new edu.asu.diging.gilesecosystem.requests.impl.Page();
-            requestPage.setPageNr(i);
+        if (pdfDocument != null) {
+            int numPages = pdfDocument.getNumberOfPages();
+            PDFRenderer renderer = new PDFRenderer(pdfDocument);
+
+            String restEndpoint = getRestEndpoint();
+
+            for (int i = 0; i < numPages; i++) {
+                edu.asu.diging.gilesecosystem.requests.impl.Page requestPage = new edu.asu.diging.gilesecosystem.requests.impl.Page();
+                requestPage.setPageNr(i);
+
+                try {
+                    BufferedImage image = renderer.renderImageWithDPI(i,
+                            Float.parseFloat(dpi), ImageType.valueOf(type));
+                    String fileName = request.getFilename() + "." + i + "." + format;
+                    Page pageImage = saveImage(request.getRequestId(),
+                            request.getDocumentId(), image, fileName);
+
+                    requestPage.setDownloadUrl(restEndpoint
+                            + DownloadFileController.GET_FILE_URL
+                                    .replace(DownloadFileController.REQUEST_ID_PLACEHOLDER,
+                                            request.getRequestId())
+                                    .replace(DownloadFileController.DOCUMENT_ID_PLACEHOLDER,
+                                            request.getDocumentId())
+                                    .replace(DownloadFileController.FILENAME_PLACEHOLDER,
+                                            pageImage.filename));
+                    requestPage.setPathToFile(pageImage.path);
+                    requestPage.setFilename(pageImage.filename);
+                    requestPage.setContentType(pageImage.contentType);
+                    requestPage.setSize(pageImage.size);
+                    requestPage.setStatus(PageStatus.COMPLETE);
+
+                } catch (IllegalArgumentException | IOException e) {
+                    logger.error("Could not render image.", e);
+                    requestPage.setStatus(PageStatus.FAILED);
+                    requestPage.setErrorMsg(e.getMessage());
+                } 
+
+                pages.add(requestPage);
+            }
 
             try {
-                BufferedImage image = renderer.renderImageWithDPI(i,
-                        Float.parseFloat(dpi), ImageType.valueOf(type));
-                String fileName = request.getFilename() + "." + i + "." + format;
-                Page pageImage = saveImage(request.getRequestId(),
-                        request.getDocumentId(), image, fileName);
-
-                requestPage.setDownloadUrl(restEndpoint
-                        + DownloadFileController.GET_FILE_URL
-                                .replace(DownloadFileController.REQUEST_ID_PLACEHOLDER,
-                                        request.getRequestId())
-                                .replace(DownloadFileController.DOCUMENT_ID_PLACEHOLDER,
-                                        request.getDocumentId())
-                                .replace(DownloadFileController.FILENAME_PLACEHOLDER,
-                                        pageImage.filename));
-                requestPage.setPathToFile(pageImage.path);
-                requestPage.setFilename(pageImage.filename);
-                requestPage.setContentType(pageImage.contentType);
-                requestPage.setSize(pageImage.size);
-                requestPage.setStatus(PageStatus.COMPLETE);
-
-            } catch (IllegalArgumentException | IOException e) {
-                logger.error("Could not render image.", e);
-                requestPage.setStatus(PageStatus.FAILED);
-                requestPage.setErrorMsg(e.getMessage());
-            } 
-
-            pages.add(requestPage);
+                pdfDocument.close();
+            } catch (IOException e) {
+                logger.error("Error closing document.", e);
+            }
         }
 
-        try {
-            pdfDocument.close();
-        } catch (IOException e) {
-            logger.error("Error closing document.", e);
-        }
 
         ICompletedImageExtractionRequest completedRequest = null;
         try {
@@ -153,7 +159,7 @@ public void extractImages(IImageExtractionRequest request)
         }
 
         completedRequest.setDocumentId(request.getDocumentId());
-        completedRequest.setStatus(RequestStatus.COMPLETE);
+        completedRequest.setStatus(status);
         completedRequest.setExtractionDate(OffsetDateTime.now(ZoneId.of("UTC"))
                 .toString());
         completedRequest.setPages(pages);
@@ -163,7 +169,7 @@ public void extractImages(IImageExtractionRequest request)
                     .sendRequest(
                             completedRequest,
                             propertiesManager
-                                    .getProperty(IPropertiesManager.KAFKA_IMAGE_EXTRACTION_COMPLETE_TOPIC));
+                                    .getProperty(Properties.KAFKA_IMAGE_EXTRACTION_COMPLETE_TOPIC));
         } catch (MessageCreationException e) {
             logger.error("Could not send message.", e);
         }
@@ -172,10 +178,10 @@ public void extractImages(IImageExtractionRequest request)
 
     private Page saveImage(String requestId, String documentId, BufferedImage image,
             String fileName) throws IOException, FileNotFoundException {
-        String dpi = propertiesManager.getProperty(IPropertiesManager.PDF_TO_IMAGE_DPI)
+        String dpi = propertiesManager.getProperty(Properties.PDF_TO_IMAGE_DPI)
                 .trim();
         String format = propertiesManager.getProperty(
-                IPropertiesManager.PDF_TO_IMAGE_FORMAT).trim();
+                Properties.PDF_TO_IMAGE_FORMAT).trim();
 
         String dirFolder = fileStorageManager.getAndCreateStoragePath(requestId,
                 documentId, null);