feat(VirusScanSchedulerService): Scheduler Service for deletion of at…

…tachment from local FS

Signed-off-by: ravi110336 <[email protected]>

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/AttachmentDatabaseHandler.java

@@ -271,4 +271,7 @@ public List<Source> getAttachmentOwnersByIds(Set<String> ids) {
     public List<AttachmentUsage> getAttachmentUsagesByReleaseId(String releaseId) {
         return attachmentUsageRepository.getUsagesByReleaseId(releaseId);
     }
+    public RequestStatus deleteOldAttachmentFromFileSystem() throws TException {
+        return DatabaseHandlerUtil.deleteOldAttachmentFromFileSystem();
+    }
 }

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/DatabaseHandlerUtil.java

@@ -9,16 +9,24 @@
  */
 package org.eclipse.sw360.datahandler.db;
 
+import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
@@ -57,6 +65,7 @@
 import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.RepositoryMixin;
 import org.eclipse.sw360.datahandler.couchdb.DatabaseMixInForChangeLog.VendorMixin;
 import org.eclipse.sw360.datahandler.thrift.ProjectReleaseRelationship;
+import org.eclipse.sw360.datahandler.thrift.RequestStatus;
 import org.eclipse.sw360.datahandler.thrift.SW360Exception;
 import org.eclipse.sw360.datahandler.thrift.attachments.Attachment;
 import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
@@ -101,13 +110,16 @@ public class DatabaseHandlerUtil {
     private static final String ATTACHMENT_STORE_FILE_SYSTEM_LOCATION;
     private static final String ATTACHMENT_STORE_FILE_SYSTEM_PERMISSION;
     private static ExecutorService ATTACHMENT_FILE_SYSTEM_STORE_THREAD_POOL = Executors.newFixedThreadPool(5);
+    private static final String ATTACHMENT_DELETE_NO_OF_DAYS;
     static {
         Properties props = CommonUtils.loadProperties(DatabaseSettings.class, PROPERTIES_FILE_PATH);
         ATTACHMENT_STORE_FILE_SYSTEM_LOCATION = props.getProperty("attachment.store.file.system.location",
                 "/opt/sw360tempattachments");
         ATTACHMENT_STORE_FILE_SYSTEM_PERMISSION = props.getProperty("attachment.store.file.system.permission",
                 "rwx------");
         IS_STORE_ATTACHMENT_TO_FILE_SYSTEM_ENABLED = Boolean.parseBoolean(props.getProperty("enable.attachment.store.to.file.system", "false"));
+        ATTACHMENT_DELETE_NO_OF_DAYS = props.getProperty("attachemnt.delete.no.of.days",
+                "30");
     }
 
     public DatabaseHandlerUtil(DatabaseConnectorCloudant db) {
@@ -795,4 +807,47 @@ public static void saveAttachmentInFileSystem(AttachmentConnector attachmentConn
                             attachmentContentId, att.getFilename());
                 });
     }
-}
+
+    public static RequestStatus deleteOldAttachmentFromFileSystem() {
+        int noOfDays = Integer.parseInt(ATTACHMENT_DELETE_NO_OF_DAYS);
+        RequestStatus status = null;
+        LocalDate todayDate = LocalDate.now();
+        LocalDate thresholdDateForAttachmentDelete = todayDate.minusDays(noOfDays);
+        Date thresholdDate = Date.from(thresholdDateForAttachmentDelete.atStartOfDay(ZoneId.systemDefault()).toInstant());
+        try {
+            deleteAttachmentAndDirectory(ATTACHMENT_STORE_FILE_SYSTEM_LOCATION, thresholdDate);
+            status = RequestStatus.SUCCESS;
+        } catch (IOException e) {
+            log.error("Unable to delete attachment. ", e);
+            status = RequestStatus.FAILURE;
+        }
+        return status;
+    }
+
+    public static void deleteAttachmentAndDirectory(String directoryFilePath, Date thresholdDate) throws IOException {
+        Path directory = Paths.get(directoryFilePath);
+        if (Files.exists(directory)) {
+            Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
+                    File file = path.toFile();
+                    long fileLastModifiedDate = file.lastModified();
+                    Date modifieddate = new Date(fileLastModifiedDate);
+                    if (thresholdDate.after(modifieddate)) {
+                        Files.delete(path);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+                @Override
+                public FileVisitResult postVisitDirectory(Path directory, IOException ioException) throws IOException {
+                    DirectoryStream<Path> stream = Files.newDirectoryStream(directory);
+                    boolean isFolderEmpty = !stream.iterator().hasNext();
+                    if (isFolderEmpty) {
+                        Files.delete(directory);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        }
+    }
+}

backend/src-common/src/main/resources/sw360.properties

@@ -72,3 +72,4 @@ textForRejectedClearingRequest= your clearing request with id: %s for the projec
 #attachment.store.file.system.location=/opt/sw360tempattachments
 #enable.attachment.store.to.file.system=false
 #attachment.store.file.system.permission=rwx------
+#attachemnt.delete.no.of.days=30

backend/src/src-attachments/src/main/java/org/eclipse/sw360/attachments/AttachmentHandler.java

@@ -243,4 +243,8 @@ public List<AttachmentUsage> getAttachmentUsagesByReleaseId(String releaseId) th
         return handler.getAttachmentUsagesByReleaseId(releaseId);
     }
 
+    @Override
+    public RequestStatus deleteOldAttachmentFromFileSystem() throws TException {
+        return handler.deleteOldAttachmentFromFileSystem();
+    }
 }

backend/src/src-projects/src/main/java/org/eclipse/sw360/projects/ProjectHandler.java

@@ -376,4 +376,4 @@ public List<Map<String, String>> getClearingStateInformationForListView(String p
         assertNotNull(projectId);
         return handler.getClearingStateInformationForListView(projectId,user);
     }
-}
+}

backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/service/ScheduleHandler.java

@@ -69,6 +69,9 @@ public RequestSummary scheduleService(String serviceName) throws TException {
             case ThriftClients.CVESEARCH_SERVICE:
                 successSync = wrapSupplierException(() -> thriftClients.makeCvesearchClient().update(), serviceName);
                 break;
+            case ThriftClients.DELETE_ATTACHMENT_SERVICE:
+                successSync = wrapSupplierException(() -> thriftClients.makeAttachmentClient().deleteOldAttachmentFromFileSystem(), serviceName);
+                break;
             default:
                 log.error("Could not schedule service: " + serviceName + ". Reason: service is not registered in ThriftClients.");
         }

backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/timer/ScheduleConstants.java

@@ -36,7 +36,10 @@ private ScheduleConstants(){}
     public static final String AUTOSTART_PROPERTY_NAME = "autostart";
     public static final String CVESEARCH_OFFSET_DEFAULT  = 0 + "" ; // default 00:00 am, in seconds
     public static final String CVESEARCH_INTERVAL_DEFAULT  = (24*60*60)+"" ; // default 24h, in seconds
-
+    public static final String DELETE_ATTACHMENT_OFFSET_DEFAULT  = "0"; // default 00:00 am, in seconds
+    public static final String DELETE_ATTACHMENT_INTERVAL_DEFAULT  = (24*60*60) + "" ; // default 24h, in seconds
+    public static final String DELETE_ATTACHMENT_OFFSET_PROPERTY_NAME = "schedule.delete.attachment.firstOffset.seconds";
+    public static final String DELETE_ATTACHMENT_INTERVAL_PROPERTY_NAME = "schedule.delete.attachment.interval.seconds";
 
     // scheduler properties
     public static final ConcurrentHashMap<String, Integer> SYNC_FIRST_RUN_OFFSET_SEC = new ConcurrentHashMap<>();
@@ -69,8 +72,29 @@ private ScheduleConstants(){}
             invalidConfiguredServices.add(ThriftClients.CVESEARCH_SERVICE);
         }
 
+        if(! props.containsKey(DELETE_ATTACHMENT_OFFSET_PROPERTY_NAME)){
+            log.debug("Property " + DELETE_ATTACHMENT_OFFSET_PROPERTY_NAME + " not set. Using default value.");
+        }
+        String deleteAttachmentOffset  = props.getProperty(DELETE_ATTACHMENT_OFFSET_PROPERTY_NAME, DELETE_ATTACHMENT_OFFSET_DEFAULT);
+        try {
+            SYNC_FIRST_RUN_OFFSET_SEC.put(ThriftClients.DELETE_ATTACHMENT_SERVICE, Integer.parseInt(deleteAttachmentOffset));
+        } catch (NumberFormatException nfe){
+            log.error("Property " + DELETE_ATTACHMENT_OFFSET_PROPERTY_NAME + " is not an integer.");
+            invalidConfiguredServices.add(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+        }
+
+        if(! props.containsKey(DELETE_ATTACHMENT_INTERVAL_PROPERTY_NAME)){
+            log.debug("Property "+ DELETE_ATTACHMENT_INTERVAL_PROPERTY_NAME + " not set. Using default value.");
+        }
+        String deleteAttachmentInterval  = props.getProperty(DELETE_ATTACHMENT_INTERVAL_PROPERTY_NAME, DELETE_ATTACHMENT_INTERVAL_DEFAULT);
+        try {
+            SYNC_INTERVAL_SEC.put(ThriftClients.DELETE_ATTACHMENT_SERVICE, Integer.parseInt(deleteAttachmentInterval));
+        } catch (NumberFormatException nfe){
+            log.error("Property " + DELETE_ATTACHMENT_INTERVAL_PROPERTY_NAME + " is not an integer.");
+            invalidConfiguredServices.add(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+        }
+
         String autostartServicesString = props.getProperty(AUTOSTART_PROPERTY_NAME, "");
         autostartServices = autostartServicesString.split(",");
     }
-
 }

backend/src/src-schedule/src/main/resources/sw360.properties

@@ -16,3 +16,7 @@ schedule.cvesearch.interval.seconds = 86400
 #general pattern for scheduling multiple services: autostart = service1,service2,service3,...
 #for scheduling the cvesearchService, uncomment the following line:
 #autostart = cvesearchService
+
+schedule.delete.attachment.firstOffset.seconds = 0
+
+schedule.delete.attachment.interval.seconds = 86400

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java

@@ -352,6 +352,11 @@ public class PortalConstants {
     public static final String CVESEARCH_INTERVAL = "cvesearchInterval";
     public static final String CVESEARCH_NEXT_SYNC = "cvesearchNextSync";
 
+    public static final String DELETE_ATTACHMENT_IS_SCHEDULED = "deleteAttachmentIsScheduled";
+    public static final String DELETE_ATTACHMENT_OFFSET = "deleteAttachmentOffset";
+    public static final String DELETE_ATTACHMENT_INTERVAL = "deleteAttachmentInterval";
+    public static final String DELETE_ATTACHMENT_NEXT_SYNC = "deleteAttachmentNextSync";
+
     //! Specialized keys for licenseInfo
     public static final String LICENSE_INFO_OUTPUT_FORMATS = "licenseInfoOutputFormats";
     public static final String LICENSE_INFO_SELECTED_OUTPUT_FORMAT = "licenseInfoSelectedOutputFormat";

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ScheduleAdminPortlet.java

@@ -74,6 +74,15 @@ private void prepareStandardView(RenderRequest request, RenderResponse response)
             request.setAttribute(PortalConstants.CVESEARCH_INTERVAL, CommonUtils.formatTime(intervalInSeconds));
             String nextSync = scheduleClient.getNextSync(ThriftClients.CVESEARCH_SERVICE);
             request.setAttribute(PortalConstants.CVESEARCH_NEXT_SYNC, nextSync);
+
+            boolean isDeleteAttachmentScheduled = isDeleteAttachmentScheduled(scheduleClient, user);
+            request.setAttribute(PortalConstants.DELETE_ATTACHMENT_IS_SCHEDULED, isDeleteAttachmentScheduled);
+            int offsetInSecondsForDeleteAttachment = scheduleClient.getFirstRunOffset(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+            request.setAttribute(PortalConstants.DELETE_ATTACHMENT_OFFSET, CommonUtils.formatTime(offsetInSecondsForDeleteAttachment));
+            int intervalInSecondsForDeleteAttachment = scheduleClient.getInterval(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+            request.setAttribute(PortalConstants.DELETE_ATTACHMENT_INTERVAL, CommonUtils.formatTime(intervalInSecondsForDeleteAttachment));
+            String nextSyncForDeleteAttachment = scheduleClient.getNextSync(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+            request.setAttribute(PortalConstants.DELETE_ATTACHMENT_NEXT_SYNC, nextSyncForDeleteAttachment);
         } catch (TException te) {
             log.error(te.getMessage());
         }
@@ -133,4 +142,60 @@ public void unscheduleAllServices(ActionRequest request, ActionResponse response
             log.error(e);
         }
     }
+
+    private boolean isDeleteAttachmentScheduled(ScheduleService.Iface scheduleClient, User user) throws TException {
+        RequestStatusWithBoolean requestStatus = scheduleClient.isServiceScheduled(ThriftClients.DELETE_ATTACHMENT_SERVICE, user);
+        if(RequestStatus.SUCCESS.equals(requestStatus.getRequestStatus())){
+            return requestStatus.isAnswerPositive();
+        } else {
+            throw new SW360Exception("Backend query for schedule status of deleteAttachment failed.");
+        }
+    }
+
+    @UsedAsLiferayAction
+    public void scheduleDeleteAttachment(ActionRequest request, ActionResponse response) throws PortletException, IOException {
+        try {
+            RequestSummary requestSummary =
+                    new ThriftClients().makeScheduleClient().scheduleService(ThriftClients.DELETE_ATTACHMENT_SERVICE);
+            setSessionMessage(request, requestSummary.getRequestStatus(), "Task", "schedule");
+        } catch (TException e) {
+            log.error("Unable to Schedule the delete attachment service. ", e);
+            e.printStackTrace();
+        }
+    }
+
+    @UsedAsLiferayAction
+    public void unscheduleDeleteAttachment(ActionRequest request, ActionResponse response) throws PortletException, IOException {
+        try {
+            User user = UserCacheHolder.getUserFromRequest(request);
+            RequestStatus requestStatus =
+                    new ThriftClients().makeScheduleClient().unscheduleService(ThriftClients.DELETE_ATTACHMENT_SERVICE, user);
+            setSessionMessage(request, requestStatus, "Task", "unschedule");
+        } catch (TException e) {
+            log.error("Unable to Unschedule the delete attachment service. ", e);
+            e.printStackTrace();
+        }
+    }
+
+    @UsedAsLiferayAction
+    public void triggerDeleteAttachment(ActionRequest request, ActionResponse response) throws PortletException, IOException {
+        try {
+            RequestStatus requestStatus = new ThriftClients().makeAttachmentClient().deleteOldAttachmentFromFileSystem();
+            setSessionMessage(request, requestStatus, "Task", "perform");
+        } catch (TException e) {
+            log.error("Unable to Manually trigger the  delete attachment service. ", e);
+            e.printStackTrace();
+        }
+    }
+
+    @UsedAsLiferayAction
+    public void triggerCveSearch(ActionRequest request, ActionResponse response) throws PortletException, IOException {
+        try {
+            RequestStatus requestStatus = new ThriftClients().makeCvesearchClient().update();
+            setSessionMessage(request, requestStatus, "Task", "perform");
+        } catch (TException e) {
+            log.error("Unable to Manually trigger the  CVE search service. ", e);
+            e.printStackTrace();
+        }
+    }
 }

frontend/sw360-portlet/src/main/resources/META-INF/resources/html/admin/scheduleAdmin/view.jsp

@@ -31,6 +31,18 @@
 <portlet:actionURL var="unscheduleAllServicesURL" name="unscheduleAllServices">
 </portlet:actionURL>
 
+<portlet:actionURL var="scheduleDeleteAttachmentURL" name="scheduleDeleteAttachment">
+</portlet:actionURL>
+
+<portlet:actionURL var="unscheduleDeleteAttachmentURL" name="unscheduleDeleteAttachment">
+</portlet:actionURL>
+
+<portlet:actionURL var="triggerDeleteAttachmentURL" name="triggerDeleteAttachment">
+</portlet:actionURL>
+
+<portlet:actionURL var="triggerCveSearchURL" name="triggerCveSearch">
+</portlet:actionURL>
+
 <div class="container">
     <div class="row portlet-toolbar">
         <div class="col-auto">
@@ -70,4 +82,40 @@
             </form>
         </div>
     </div>
+    <div class="row">
+        <div class="col-6">
+            <h4><liferay-ui:message key="attachment.deletion.from.local.fs" /></h4>
+             <table class="table bordered-table">
+                <tr>
+                    <th><liferay-ui:message key="schedule.offset" /></th>
+                     <td><sw360:out value="${deleteAttachmentOffset} (hh:mm:ss)" /></td>
+                </tr>
+                <tr>
+                    <th><liferay-ui:message key="interval" /></th>
+                    <td><sw360:out value="${deleteAttachmentInterval} (hh:mm:ss)" /></td>
+                </tr>
+                <tr>
+                    <th><liferay-ui:message key="next.synchronization" /></th>
+                     <td><sw360:out value="${deleteAttachmentNextSync}"/></td>
+                </tr>
+            </table>
+            <form class="form mt-3">
+                <div class="form-group">
+                    <button type="button" class="btn btn-primary" onclick="window.location.href='<%=scheduleDeleteAttachmentURL%>'"<core_rt:if test="${deleteAttachmentIsScheduled}">disabled</core_rt:if>><liferay-ui:message key="schedule.attachment.deletion.from.local.fs" /></button>
+                    <button type="button" class="btn btn-light" onclick="window.location.href='<%=unscheduleDeleteAttachmentURL%>'"<core_rt:if test="${not deleteAttachmentIsScheduled}">disabled</core_rt:if>><liferay-ui:message key="cancel.scheduled.attachment.deletion.from.local.fs" /></button>
+                </div>
+            </form>
+        </div>
+    </div>
+    <div class="row">
+        <div>
+            <h4><liferay-ui:message key="manual.triggering.of.scheduled.services" /></h4>
+            <form class="form mt-3">
+                <div class="form-group">
+                    <button type="button" class="btn btn-primary" onclick="window.location.href='<%=triggerDeleteAttachmentURL%>'"><liferay-ui:message key="attachment.deletion.from.local.fs" /></button>
+                    <button type="button" class="btn btn-primary" onclick="window.location.href='<%=triggerCveSearchURL%>'"><liferay-ui:message key="cve.search" /></button>
+                </div>
+            </form>
+        </div>
+    </div>
 </div>

frontend/sw360-portlet/src/main/resources/content/Language.properties

@@ -90,6 +90,7 @@ AttachmentType-SOURCE=original course code
 AttachmentType-SOURCE_SELF=Self assembled source code distribution
 attachment.usages=Attachment Usages
 attachment.usages2=attachment usage(s)
+attachment.deletion.from.local.fs=Attachment Deletion From Local FS
 attribute.changes=Attribute Changes
 ausfuhrliste=Ausfuhrliste
 authorities=Authorities
@@ -115,6 +116,7 @@ by.clicking.on.the.checkbox.in.the.header.row=by clicking on the checkbox in the
 cancel=Cancel
 cancel.all.scheduled.tasks=Cancel all Scheduled Tasks
 cancel.cve.service=Cancel CVE service
+cancel.scheduled.attachment.deletion.from.local.fs=Cancel Scheduled Attachment Deletion From Local FS
 cannot.add.attachments.before.saving.the.document=Cannot add attachments before saving the document
 cannot.continue.to.merge.of.releases=Cannot continue to merge of releases: 
 can.not.get.products=Can not get products!
@@ -696,6 +698,7 @@ MainlineState-MAINLINE=Organisation or person thinks that use of this software i
 MainlineState-OPEN=Not decided so far
 MainlineState-PHASEOUT=The software has issues, please consider removing it soon, if in use.
 MainlineState-SPECIFIC=The software is not recommended in general, but for special use case or for this particular version it is acceptable.
+manual.triggering.of.scheduled.services=Manual triggering of scheduled services
 matched.by=Matched by
 matching.source.attachments=Matching Source Attachments
 material.index.number=Material Index Number
@@ -1049,6 +1052,7 @@ scanned=Scanned
 scan.the.sources=Scan the sources
 schedule=Schedule
 schedule.cve.service=Schedule CVE service
+schedule.attachment.deletion.from.local.fs=Schedule Attachment Deletion From Local FS
 schedule.offset=Schedule Offset
 schedule.task.administration=Schedule Task Administration
 scope=Scope