Skip to content

Commit

Permalink
EPMRPP-87504 || Remove empty directories after removing attachments a…
Browse files Browse the repository at this point in the history 
…nd fix DeleteExpiredUsersJob (#114)
  • Loading branch information
@IvanKustau
IvanKustau authored Nov 11, 2023
1 parent 3f0d306 commit 56fa3b2
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 45 deletions.
5 changes: 3 additions & 2 deletions src/main/java/com/epam/reportportal/config/DataStorageConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,9 @@ public BlobStore filesystemBlobStore(
@Bean
@ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem")
public DataStorageService localDataStore(@Autowired BlobStore blobStore,
FeatureFlagHandler featureFlagHandler) {
return new LocalDataStorageService(blobStore, featureFlagHandler);
FeatureFlagHandler featureFlagHandler,
@Value("${datastore.path:/data/store}") String baseDirectory) {
return new LocalDataStorageService(blobStore, featureFlagHandler, baseDirectory);
}

/**
Expand Down
29 changes: 2 additions & 27 deletions src/main/java/com/epam/reportportal/jobs/clean/DeleteExpiredUsersJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,15 @@
import com.epam.reportportal.model.activity.event.UnassignUserEvent;
import com.epam.reportportal.model.activity.event.UserDeletedEvent;
import com.epam.reportportal.service.MessageBus;
import com.epam.reportportal.storage.DataStorageService;
import com.epam.reportportal.utils.FeatureFlag;
import com.epam.reportportal.utils.FeatureFlagHandler;
import com.epam.reportportal.utils.ValidationUtil;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -113,25 +107,18 @@ WHERE id IN (
@Value("${rp.environment.variable.clean.expiredUser.retentionPeriod}")
private Long retentionPeriod;

private final DataStorageService dataStorageService;

private final IndexerServiceClient indexerServiceClient;

private final FeatureFlagHandler featureFlagHandler;

private final MessageBus messageBus;

@Autowired
public DeleteExpiredUsersJob(JdbcTemplate jdbcTemplate,
NamedParameterJdbcTemplate namedParameterJdbcTemplate, DataStorageService dataStorageService,
IndexerServiceClient indexerServiceClient, MessageBus messageBus,
FeatureFlagHandler featureFlagHandler) {
NamedParameterJdbcTemplate namedParameterJdbcTemplate,
IndexerServiceClient indexerServiceClient, MessageBus messageBus) {
super(jdbcTemplate);
this.namedParameterJdbcTemplate = namedParameterJdbcTemplate;
this.dataStorageService = dataStorageService;
this.indexerServiceClient = indexerServiceClient;
this.messageBus = messageBus;
this.featureFlagHandler = featureFlagHandler;
}

@Override
Expand Down Expand Up @@ -197,13 +184,6 @@ private void deleteProjectAssociatedData(Long projectId) {
deleteAttachmentsByProjectId(projectId);
deleteProjectIssueTypes(projectId);
indexerServiceClient.removeSuggest(projectId);
try {
if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
dataStorageService.deleteContainer(projectId.toString());
}
} catch (Exception e) {
LOGGER.warn("Cannot delete attachments bucket for project {} ", projectId);
}
indexerServiceClient.deleteIndex(projectId);
}

Expand Down Expand Up @@ -284,9 +264,4 @@ public void setProjectId(long projectId) {
this.projectId = projectId;
}
}

private String decode(String data) {
return StringUtils.isEmpty(data) ? data :
new String(Base64.getUrlDecoder().decode(data), StandardCharsets.UTF_8);
}
}
86 changes: 70 additions & 16 deletions src/main/java/com/epam/reportportal/storage/LocalDataStorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

import com.epam.reportportal.utils.FeatureFlag;
import com.epam.reportportal.utils.FeatureFlagHandler;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand All @@ -40,11 +43,17 @@ public class LocalDataStorageService implements DataStorageService {

private final FeatureFlagHandler featureFlagHandler;

private final String baseDirectory;

private static final String PROJECT_PREFIX = "project-data";

private static final String SINGLE_BUCKET_NAME = "store";

public LocalDataStorageService(BlobStore blobStore, FeatureFlagHandler featureFlagHandler) {
public LocalDataStorageService(BlobStore blobStore, FeatureFlagHandler featureFlagHandler,
String baseDirectory) {
this.blobStore = blobStore;
this.featureFlagHandler = featureFlagHandler;
this.baseDirectory = baseDirectory;
}

@Override
Expand All @@ -53,24 +62,17 @@ public void deleteAll(List<String> paths) throws Exception {
return;
}
if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
removeFiles(SINGLE_BUCKET_NAME, paths);
} else {
Map<String, List<String>> bucketPathMap = new HashMap<>();
for (String path : paths) {
Path targetPath = Paths.get(path);
int nameCount = targetPath.getNameCount();
String bucket = retrievePath(targetPath, 0, 1);
String cutPath = retrievePath(targetPath, 1, nameCount);
if (bucketPathMap.containsKey(bucket)) {
bucketPathMap.get(bucket).add(cutPath);
} else {
List<String> bucketPaths = new ArrayList<>();
bucketPaths.add(cutPath);
bucketPathMap.put(bucket, bucketPaths);
}
Map<String, List<String>> bucketPathMap = retrieveBucketPathMap(paths);
for (Map.Entry<String, List<String>> bucketPaths : bucketPathMap.entrySet()) {
removeFiles(SINGLE_BUCKET_NAME, bucketPaths.getValue());
deleteEmptyDirs(
Paths.get(baseDirectory, SINGLE_BUCKET_NAME, PROJECT_PREFIX, bucketPaths.getKey()));
}
} else {
Map<String, List<String>> bucketPathMap = retrieveBucketPathMap(paths);
for (Map.Entry<String, List<String>> bucketPaths : bucketPathMap.entrySet()) {
removeFiles(bucketPaths.getKey(), bucketPaths.getValue());
deleteEmptyDirs(Paths.get(baseDirectory, bucketPaths.getKey()));
}
}
}
Expand All @@ -84,6 +86,25 @@ public void deleteContainer(String containerName) {
}
}

private Map<String, List<String>> retrieveBucketPathMap(List<String> paths) {
Map<String, List<String>> bucketPathMap = new HashMap<>();
for (String path : paths) {
Path targetPath = Paths.get(path);
int nameCount = targetPath.getNameCount();
String bucket = retrievePath(targetPath, 0, 1);
String cutPath = retrievePath(targetPath, 1, nameCount);
if (bucketPathMap.containsKey(bucket)) {
bucketPathMap.get(bucket).add(cutPath);
} else {
List<String> bucketPaths = new ArrayList<>();
bucketPaths.add(cutPath);
bucketPathMap.put(bucket, bucketPaths);
}
}

return bucketPathMap;
}

private String retrievePath(Path path, int beginIndex, int endIndex) {
return String.valueOf(path.subpath(beginIndex, endIndex));
}
Expand All @@ -96,4 +117,37 @@ private void removeFiles(String bucketName, List<String> paths) {
}
}

private void deleteEmptyDirs(Path dir) {
if (!Files.isDirectory(dir)) {
return;
}

// List all files/directories in the given directory
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path entry : stream) {
deleteEmptyDirs(entry);
}
} catch (IOException e) {
LOGGER.warn("Exception {} is occurred during checking directory", e.getMessage());
}

// Delete the directory if empty
try {
if (isDirectoryEmpty(dir)) {
Files.delete(dir);
}
} catch (IOException e) {
LOGGER.warn("Exception {} is occurred during deleting empty directory", e.getMessage());
}
}

private boolean isDirectoryEmpty(Path dir) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
return !stream.iterator().hasNext();
} catch (IOException e) {
LOGGER.warn("Exception {} is occurred during checking directory", e.getMessage());
return false;
}
}

}

0 comments on commit 56fa3b2

Please sign in to comment.