added mode for storing signatures to files

Cornul11 · Mar 2, 2024 · 0f1731f · 0f1731f
1 parent 9360ef6
commit 0f1731f
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 6 deletions.
diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/CommandExecutor.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/CommandExecutor.java
@@ -11,8 +11,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public class CommandExecutor {
     private static final Logger logger = LoggerFactory.getLogger(CommandExecutor.class);
@@ -45,7 +50,7 @@ public void run() {
 
             switch (mode) {
                 case "CORPUS_GEN_MODE":
-                    JarFileExplorer jarFileExplorer = new JarFileExplorer(signatureDao, config);
+                    JarFileExplorer jarFileExplorer = new JarFileExplorer(signatureDao, config, null);
                     logger.info("Starting CORPUS_GEN_MODE; ignoring uber JARs: " + config.getIgnoreUberJarSignatures());
                     String directoryPath = options.getDirectory();
                     String filePaths = options.getFilePaths();
@@ -89,6 +94,33 @@ public void run() {
                         printHelpMessage();
                     }
                     break;
+                case "EXTRACT_SIGNATURES":
+                    String directoryPathForSignatures = options.getDirectory();
+                    String filePathsForSignatures = options.getFilePaths();
+                    String basePath = options.getBasePath();
+                    if (directoryPathForSignatures == null || filePathsForSignatures == null || basePath == null) {
+                        System.out.println("Directory path, file path(s) and base path is required for EXTRACT_SIGNATURES");
+                        printHelpMessage();
+                    } else {
+                        List<Path> jarFilePaths = null;
+                        Path filePath = Paths.get(filePathsForSignatures);
+                        try {
+                            List<String> lines = Files.readAllLines(filePath);
+                            jarFilePaths = lines.stream().map(Paths::get).collect(Collectors.toList());
+                        } catch (IOException e) {
+                            logger.error("Error reading file: " + filePath, e);
+                        }
+
+                        if (jarFilePaths == null) {
+                            System.out.println("No jar file paths found in the file: " + filePathsForSignatures);
+                            return;
+                        }
+                        JarFileExplorer jarFileExplorerForSignatures = new JarFileExplorer(signatureDao, config, directoryPathForSignatures);
+
+                        jarFileExplorerForSignatures.processFilesToFiles(jarFilePaths, basePath);
+
+                    }
+                    break;
                 default:
                     System.out.println("Invalid mode specified: " + mode);
                     printHelpMessage();

diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/OptionsBuilder.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/OptionsBuilder.java
@@ -31,6 +31,10 @@ public String getDirectory() {
         return cmd.getOptionValue("d");
     }
 
+    public String getBasePath() {
+        return cmd.getOptionValue("bp");
+    }
+
     public String getEvaluationDirectory() {
         return cmd.getOptionValue("e");
     }
@@ -114,6 +118,13 @@ private Options buildOptions() {
                 .desc("Specify the path to a file containing paths for processing")
                 .build());
 
+        options.addOption(Option.builder("bp")
+                .longOpt("basePath")
+                .hasArg()
+                .argName("path")
+                .desc("Specify the base path for where the JARs are located")
+                .build());
+
         options.addOption(Option.builder("e")
                 .longOpt("evaluationDirectory")
                 .hasArg()

diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/FileAnalyzer.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/FileAnalyzer.java
@@ -9,7 +9,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
@@ -28,12 +31,36 @@ public class FileAnalyzer {
     private final int totalJars;
     private final long startTime = System.currentTimeMillis();
     private final String basePath;
+    private final Path outputDirectory;
 
-    public FileAnalyzer(SignatureDAO signatureDao, ConfigurationLoader config) {
+    public FileAnalyzer(SignatureDAO signatureDao, ConfigurationLoader config, String outputDirectory) {
         this.config = config;
         this.signatureDao = signatureDao;
         this.totalJars = config.getTotalJars();
         this.basePath = config.getBasePath();
+        this.outputDirectory = outputDirectory == null ? null : Path.of(outputDirectory);
+    }
+
+    public void processJarFileToFile(Path jarFilePath, String basePath) {
+        JarAndPomInfoExtractor extractor = new JarAndPomInfoExtractor(jarFilePath.toString(), basePath);
+        String groupId = extractor.getGroupId();
+        String artifactId = extractor.getArtifactId();
+        String version = extractor.getVersion();
+
+        JarHandler jarHandler = new JarHandler(jarFilePath, ignoredUberJars, insertedLibraries, config);
+        List<String> classHashes = jarHandler.extractSignatures().stream().map(signature -> Long.toString(signature.getHashCode())).collect(Collectors.toList());
+
+        String groupPath = groupId.replace('.', '/');
+        Path outputFile = Paths.get(outputDirectory.toString(), groupPath, artifactId, version, artifactId + "-" + version + ".jar.txt");
+
+        try {
+            Files.createDirectories(outputFile.getParent());
+
+            Files.write(outputFile, classHashes);
+            logger.info("Saved class hashes for " + jarFilePath.getFileName().toString());
+        } catch (IOException e) {
+            logger.error("Error writing to file: " + outputFile, e);
+        }
     }
 
     public void printIgnoredUberJars() {
@@ -121,8 +148,8 @@ private void calculateAndLogElapsedTime() {
 
         logger.info(String.format("Done processing %d/%d JARs, progress: \u001B[94m%d%%\u001B[0m, ETA: %d days, %d hours, %d minutes and %d seconds",
                 processed, totalJars, (processed * 100 / totalJars), etaDays, etaHours, etaMins, etaSecs));
-
     }
+
     public void printStats() {
         // TODO: investigate why the number of unique hashes is not constant for a constant given set of JARs
         logger.info("Total number of unique hashes: " + uniqueHashes.size());

diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/JarFileExplorer.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/JarFileExplorer.java
@@ -25,12 +25,30 @@ public class JarFileExplorer {
     private static final Path POISON_PILL = Paths.get("");
     private final SignatureDAO signatureDao;
 
-    public JarFileExplorer(SignatureDAO signatureDao, ConfigurationLoader config) {
+    public JarFileExplorer(SignatureDAO signatureDao, ConfigurationLoader config, String outputPath) {
         this.signatureDao = signatureDao;
-        this.fileAnalyzer = new FileAnalyzer(signatureDao, config);
+        this.fileAnalyzer = new FileAnalyzer(signatureDao, config, outputPath);
         this.numConsumerThreads = config.getNumConsumerThreads();
     }
 
+    public void processFilesToFiles(List<Path> jarFilePaths, String basePath) {
+        ExecutorService executor = Executors.newFixedThreadPool(numConsumerThreads);
+        for (Path jarFilePath : jarFilePaths) {
+            executor.submit(() -> fileAnalyzer.processJarFileToFile(jarFilePath, basePath));
+        }
+        executor.shutdown();
+        while (!executor.isTerminated()) {
+            try {
+                executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+            } catch (InterruptedException e) {
+                List<Runnable> remainingTasks = executor.shutdownNow(); // Force remaining tasks to terminate
+                logger.error("Interrupted while waiting for tasks to complete", e);
+                Thread.currentThread().interrupt();
+            }
+        }
+        logger.info("All JAR files have been processed");
+    }
+
     public void processFiles(String path, String lastPath) {
         Path rootPath = Paths.get(path);
         Path lastVisitedPath = lastPath != null ? Paths.get(lastPath) : null;

diff --git a/src/test/java/BytecodeDetailsUberJarDetectionTest.java b/src/test/java/BytecodeDetailsUberJarDetectionTest.java
@@ -41,7 +41,7 @@ public void setUp() {
         MockitoAnnotations.openMocks(this);
 
         Mockito.when(config.getIgnoreUberJarSignatures()).thenReturn(true);
-        fileAnalyzer = new FileAnalyzer(signatureDao, config);
+        fileAnalyzer = new FileAnalyzer(signatureDao, config, null);
     }
 
     @Test