From 56fa3b22ee654a654ad4b61cc8f2e9beff0d66d0 Mon Sep 17 00:00:00 2001 From: Ivan Kustau <86599591+IvanKustau@users.noreply.github.com> Date: Sat, 11 Nov 2023 12:24:52 +0300 Subject: [PATCH] EPMRPP-87504 || Remove empty directories after removing attachments and fix DeleteExpiredUsersJob (#114) --- .../config/DataStorageConfig.java | 5 +- .../jobs/clean/DeleteExpiredUsersJob.java | 29 +------ .../storage/LocalDataStorageService.java | 86 +++++++++++++++---- 3 files changed, 75 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java index 55d9478..fa88bc0 100644 --- a/src/main/java/com/epam/reportportal/config/DataStorageConfig.java +++ b/src/main/java/com/epam/reportportal/config/DataStorageConfig.java @@ -153,8 +153,9 @@ public BlobStore filesystemBlobStore( @Bean @ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem") public DataStorageService localDataStore(@Autowired BlobStore blobStore, - FeatureFlagHandler featureFlagHandler) { - return new LocalDataStorageService(blobStore, featureFlagHandler); + FeatureFlagHandler featureFlagHandler, + @Value("${datastore.path:/data/store}") String baseDirectory) { + return new LocalDataStorageService(blobStore, featureFlagHandler, baseDirectory); } /** diff --git a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java index 1c1a367..ac7fc1e 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java @@ -23,21 +23,15 @@ import com.epam.reportportal.model.activity.event.UnassignUserEvent; import com.epam.reportportal.model.activity.event.UserDeletedEvent; import com.epam.reportportal.service.MessageBus; -import com.epam.reportportal.storage.DataStorageService; -import com.epam.reportportal.utils.FeatureFlag; -import com.epam.reportportal.utils.FeatureFlagHandler; import com.epam.reportportal.utils.ValidationUtil; -import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -113,25 +107,18 @@ WHERE id IN ( @Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}") private Long retentionPeriod; - private final DataStorageService dataStorageService; - private final IndexerServiceClient indexerServiceClient; - private final FeatureFlagHandler featureFlagHandler; - private final MessageBus messageBus; @Autowired public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate, - NamedParameterJdbcTemplate namedParameterJdbcTemplate, DataStorageService dataStorageService, - IndexerServiceClient indexerServiceClient, MessageBus messageBus, - FeatureFlagHandler featureFlagHandler) { + NamedParameterJdbcTemplate namedParameterJdbcTemplate, + IndexerServiceClient indexerServiceClient, MessageBus messageBus) { super(jdbcTemplate); this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; - this.dataStorageService = dataStorageService; this.indexerServiceClient = indexerServiceClient; this.messageBus = messageBus; - this.featureFlagHandler = featureFlagHandler; } @Override @@ -197,13 +184,6 @@ private void deleteProjectAssociatedData(Long projectId) { deleteAttachmentsByProjectId(projectId); deleteProjectIssueTypes(projectId); indexerServiceClient.removeSuggest(projectId); - try { - if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { - dataStorageService.deleteContainer(projectId.toString()); - } - } catch (Exception e) { - LOGGER.warn("Cannot delete attachments bucket for project {} ", projectId); - } indexerServiceClient.deleteIndex(projectId); } @@ -284,9 +264,4 @@ public void setProjectId(long projectId) { this.projectId = projectId; } } - - private String decode(String data) { - return StringUtils.isEmpty(data) ? data : - new String(Base64.getUrlDecoder().decode(data), StandardCharsets.UTF_8); - } } diff --git a/src/main/java/com/epam/reportportal/storage/LocalDataStorageService.java b/src/main/java/com/epam/reportportal/storage/LocalDataStorageService.java index 0d49d54..9372fed 100644 --- a/src/main/java/com/epam/reportportal/storage/LocalDataStorageService.java +++ b/src/main/java/com/epam/reportportal/storage/LocalDataStorageService.java @@ -18,6 +18,9 @@ import com.epam.reportportal.utils.FeatureFlag; import com.epam.reportportal.utils.FeatureFlagHandler; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -40,11 +43,17 @@ public class LocalDataStorageService implements DataStorageService { private final FeatureFlagHandler featureFlagHandler; + private final String baseDirectory; + + private static final String PROJECT_PREFIX = "project-data"; + private static final String SINGLE_BUCKET_NAME = "store"; - public LocalDataStorageService(BlobStore blobStore, FeatureFlagHandler featureFlagHandler) { + public LocalDataStorageService(BlobStore blobStore, FeatureFlagHandler featureFlagHandler, + String baseDirectory) { this.blobStore = blobStore; this.featureFlagHandler = featureFlagHandler; + this.baseDirectory = baseDirectory; } @Override @@ -53,24 +62,17 @@ public void deleteAll(List paths) throws Exception { return; } if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { - removeFiles(SINGLE_BUCKET_NAME, paths); - } else { - Map> bucketPathMap = new HashMap<>(); - for (String path : paths) { - Path targetPath = Paths.get(path); - int nameCount = targetPath.getNameCount(); - String bucket = retrievePath(targetPath, 0, 1); - String cutPath = retrievePath(targetPath, 1, nameCount); - if (bucketPathMap.containsKey(bucket)) { - bucketPathMap.get(bucket).add(cutPath); - } else { - List bucketPaths = new ArrayList<>(); - bucketPaths.add(cutPath); - bucketPathMap.put(bucket, bucketPaths); - } + Map> bucketPathMap = retrieveBucketPathMap(paths); + for (Map.Entry> bucketPaths : bucketPathMap.entrySet()) { + removeFiles(SINGLE_BUCKET_NAME, bucketPaths.getValue()); + deleteEmptyDirs( + Paths.get(baseDirectory, SINGLE_BUCKET_NAME, PROJECT_PREFIX, bucketPaths.getKey())); } + } else { + Map> bucketPathMap = retrieveBucketPathMap(paths); for (Map.Entry> bucketPaths : bucketPathMap.entrySet()) { removeFiles(bucketPaths.getKey(), bucketPaths.getValue()); + deleteEmptyDirs(Paths.get(baseDirectory, bucketPaths.getKey())); } } } @@ -84,6 +86,25 @@ public void deleteContainer(String containerName) { } } + private Map> retrieveBucketPathMap(List paths) { + Map> bucketPathMap = new HashMap<>(); + for (String path : paths) { + Path targetPath = Paths.get(path); + int nameCount = targetPath.getNameCount(); + String bucket = retrievePath(targetPath, 0, 1); + String cutPath = retrievePath(targetPath, 1, nameCount); + if (bucketPathMap.containsKey(bucket)) { + bucketPathMap.get(bucket).add(cutPath); + } else { + List bucketPaths = new ArrayList<>(); + bucketPaths.add(cutPath); + bucketPathMap.put(bucket, bucketPaths); + } + } + + return bucketPathMap; + } + private String retrievePath(Path path, int beginIndex, int endIndex) { return String.valueOf(path.subpath(beginIndex, endIndex)); } @@ -96,4 +117,37 @@ private void removeFiles(String bucketName, List paths) { } } + private void deleteEmptyDirs(Path dir) { + if (!Files.isDirectory(dir)) { + return; + } + + // List all files/directories in the given directory + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path entry : stream) { + deleteEmptyDirs(entry); + } + } catch (IOException e) { + LOGGER.warn("Exception {} is occurred during checking directory", e.getMessage()); + } + + // Delete the directory if empty + try { + if (isDirectoryEmpty(dir)) { + Files.delete(dir); + } + } catch (IOException e) { + LOGGER.warn("Exception {} is occurred during deleting empty directory", e.getMessage()); + } + } + + private boolean isDirectoryEmpty(Path dir) { + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + return !stream.iterator().hasNext(); + } catch (IOException e) { + LOGGER.warn("Exception {} is occurred during checking directory", e.getMessage()); + return false; + } + } + }