New flow

.gitignore

@@ -117,4 +117,10 @@ nimrod/tests/env-config.json
 *.class
 
 #Microsoft
-.vscode/
+*.vscode/
+
+#vim
+*.swp
+
+#ctags
+tags

nimrod/nimrod.py

@@ -10,16 +10,19 @@
 from nimrod.tools.junit import JUnit, JUnitResult, Coverage
 from nimrod.tools.evosuite import Evosuite
 from nimrod.utils import package_to_dir
+from nimrod.tool_suite_executor import SuiteFactory, SuiteEnv
 
 from collections import namedtuple
 
 
 OUTPUT_DIR = 'nimrod_output'
 
 
-NimrodResult = namedtuple('NimrodResult', ['maybe_equivalent', 'not_equivalent',
+NimrodResult = namedtuple('NimrodResult', ['maybe_equivalent', 
+                                           'not_equivalent',
                                            'coverage', 'differential',
-                                           'timeout', 'test_tool', 'is_equal_coverage'])
+                                           'timeout', 'test_tool', 
+                                           'is_equal_coverage'])
 
 
 class Nimrod:
@@ -34,14 +37,31 @@ def __init__(self, java, maven):
     def run(self, project_dir, mutants_dir, sut_class, randoop_params=None,
             evosuite_diff_params=None, evosuite_params=None, output_dir=None):
 
-        results = {}
-
         _, classes_dir = self.maven.compile(project_dir, clean=True)
         output_dir = self.check_output_dir(output_dir if output_dir
                                            else os.path.join(project_dir,
                                                              OUTPUT_DIR))
 
-        for mutant in self.get_mutants(classes_dir, mutants_dir):
+        suite_env = SuiteEnv(
+            classes_dir=classes_dir, 
+            sut_class=sut_class, 
+            tests_src=os.path.join(output_dir, 'suites', 'ORIGINAL')
+        ) 
+
+        suites = self.generate_suites(
+            suite_env=suite_env, 
+            evo_params=evosuite_params,
+            ran_params=randoop_params
+        )  
+
+        mutants = self.get_mutants(classes_dir, mutants_dir)
+
+        mutants, results = self.try_with_original_suites(mutants, sut_class, 
+                                                        suites, output_dir)
+
+        print("\n\nFollowing the game with {0} mutans...\n"
+              .format(len(mutants)))
+        for mutant in mutants:
             start_time = time.time()
             if self.check_mutant(mutant, sut_class):
                 tests_src = os.path.join(output_dir, 'suites', mutant.mid)
@@ -106,13 +126,48 @@ def run(self, project_dir, mutants_dir, sut_class, randoop_params=None,
 
         return results
 
+    def generate_suites(self, suite_env, evo_params, ran_params):
+        suites = []
+        suite_factory = SuiteFactory(self.java, suite_env)
+
+        suites.append(suite_factory.evosuite_generate(evo_params))
+        suites.append(suite_factory.randoop_generate(ran_params))
+
+        return suites
+
+    def try_with_original_suites(self, mutants, sut_class, suites, output_dir):
+        not_killed = []
+        results = {}
+
+        for mutant in mutants:
+            if self.check_mutant(mutant, sut_class):
+                start_time = time.time()
+                killed = False
+                for suite in suites:
+                    if not killed:
+                        test_result = suite.run_with_mutant(mutant)
+                        if test_result.fail_tests > 0 or test_result.timeout:
+                            results[mutant.mid] = self.create_nimrod_result(
+                                test_result, False, suite.get_tool_name())
+                            killed = True
+                if not killed:
+                    not_killed.append(mutant)  
+                    print('\tNot killed, follow the game.')
+                else:
+                    exec_time = time.time() - start_time
+                    self.print_result(mutant, results[mutant.mid])
+                    self.write_to_csv(results[mutant.mid], mutant, output_dir, 
+                    exec_time=exec_time)                        
+
+        return not_killed, results
+
     @staticmethod
     def print_result( mutant, result):
         if result.maybe_equivalent:
-            print('{0} maybe equivalent, executions: {1}.'
+            print('\t{0} maybe equivalent, executions: {1}.'
                   .format(mutant.mid, result.coverage.executions))
         else:
-            print('{0} is not equivalent. {1}'
+            print('\t{0} is not equivalent. {1}'
                   .format(mutant.mid, 'Killed by differential test.' if
                           result.differential else ''))
 
@@ -123,12 +178,12 @@ def check_mutant(mutant, sut_class):
             class_file = os.path.join(mutant.dir, package_to_dir(sut_class) +
                                       '.class')
             if not os.path.exists(class_file):
-                print("{0}: .class not found.".format(mutant.mid))
+                print("\t{0}: .class not found.".format(mutant.mid))
                 return False
             else:
                 return True
         else:
-            print('{0}: directory not found.'.format(mutant.mid))
+            print('\t{0}: directory not found.'.format(mutant.mid))
             return False
 
     def get_mutants(self, classpath, mutants_dir):

nimrod/tests/test_utils.py

@@ -9,6 +9,7 @@
 
 from nimrod.utils import get_files, get_class_files, get_java_files
 from nimrod.utils import generate_classpath
+from nimrod.utils import dir_to_package
 from nimrod.tools.java import Java
 from nimrod.tools.maven import Maven
 
@@ -94,5 +95,11 @@ def test_generate_classpath_with_invalid_paths(self):
         self.assertEqual('b', path_split[1])
         self.assertEqual('c', path_split[2])
 
+    def test_dir_to_package(self):
+        directory = os.path.join('org', 'apache', 'commons', 'math')
+        package = dir_to_package(directory)
+
+        self.assertEqual('org.apache.commons.math', package)
+
     def tearDown(self):
         calculator_clean_project()

nimrod/tests/tools/test_java.py

@@ -7,6 +7,9 @@
 from nimrod.tests.utils import calculator_java_file
 from nimrod.tests.utils import calculator_package_dir
 from nimrod.tests.utils import calculator_operation_java_file
+from nimrod.tests.utils import calculator_src_dir
+from nimrod.utils import get_java_files
+from nimrod.utils import get_class_files
 
 
 class TestJava(TestCase):
@@ -85,3 +88,28 @@ def test_get_env(self):
         self.assertTrue('JAVA_HOME' in env)
         self.assertTrue('VARIABLE' in env)
         self.assertEqual(env['VARIABLE'], 'VALUE')
+
+    def test_compile_all(self):
+        java = Java(self.java_home)
+        src_dir = calculator_src_dir()
+
+        self._clean_class_files(src_dir)
+
+        java.compile_all(src_dir, src_dir)
+
+        class_files = 0
+
+        for java_file in get_java_files(src_dir):
+            class_file = os.path.join(src_dir,
+                                      java_file.replace('.java', '.class'))
+            if os.path.exists(class_file):
+                class_files += 1
+
+        self.assertEqual(6, class_files)
+
+        self._clean_class_files(src_dir)
+
+    @staticmethod
+    def _clean_class_files(directory):
+        for file in get_class_files(directory):
+            os.remove(os.path.join(directory, file))

nimrod/tests/tools/test_junit.py

@@ -51,9 +51,15 @@ def test_run_junit(self):
     def test_run_junit_timeout(self):
         junit = JUnit(self.java, self.classpath)
 
-        with self.assertRaises(subprocess.TimeoutExpired):
-            junit.exec(self.suite_dir, self.suite_classes_dir,
-                       self.sut_class, self.suite_classes[0], 0)
+        result = junit.exec(self.suite_dir, self.suite_classes_dir,
+                            self.sut_class, self.suite_classes[0], 0)
+
+        self.assertEqual(0, result.ok_tests)
+        self.assertEqual(0, result.fail_tests)
+        self.assertEqual(0, len(result.fail_test_set))
+        self.assertEqual(0, result.run_time)
+        self.assertIsNone(result.coverage)
+        self.assertTrue(result.timeout)
 
     def test_run_junit_with_mutant(self):
         junit = JUnit(self.java, self.classpath)

nimrod/tests/tools/test_suite_generator.py

@@ -0,0 +1,49 @@
+import os
+import shutil
+import subprocess
+
+from unittest import TestCase
+
+from nimrod.tests.utils import calculator_project_dir
+from nimrod.tests.utils import get_config
+
+from nimrod.utils import generate_classpath
+from nimrod.tools.suite_generator import SuiteGenerator
+from nimrod.tools.java import Java
+
+
+class ToolSuiteGenerator(SuiteGenerator):
+
+    def _get_tool_name(self):
+        return 'test'
+
+    def _exec_tool(self):
+        return self._exec(*tuple('-version'))
+
+    def _test_classes(self):
+        return []
+
+    @staticmethod
+    def _get_timeout():
+        return 0
+
+
+class TestSuiteGenerator(TestCase):
+
+    def setUp(self):
+        self.java = Java(get_config()['java_home'])
+
+        self.tests_src = os.path.join(calculator_project_dir(), 'test_tool')
+        self.suite_tool = ToolSuiteGenerator(self.java, '', self.tests_src, '')
+
+    def test_get_tool_name(self):
+        self.assertEqual('test', self.suite_tool._get_tool_name())
+
+    def test_generate_timeout(self):
+        try:
+            self.suite_tool.generate()
+        except subprocess.TimeoutExpired:
+            self.fail()
+
+    def tearDown(self):
+        shutil.rmtree(self.tests_src)

nimrod/tool_suite_executor.py

@@ -0,0 +1,50 @@
+from nimrod.tools.evosuite import Evosuite
+from nimrod.tools.randoop import Randoop
+from nimrod.tools.junit import JUnit
+
+from collections import namedtuple
+
+SuiteEnv = namedtuple('SuiteEnv', ['classes_dir', 'sut_class', 'tests_src'])
+
+class SuiteFactory():
+
+    def __init__(self, java, suite_env):
+        self.suite_env = suite_env
+        self.java = java
+
+    def randoop_generate(self, tool_params=None):
+        randoop = Randoop(
+            java=self.java,
+            classpath=self.suite_env.classes_dir,
+            tests_src=self.suite_env.tests_src,
+            sut_class=self.suite_env.sut_class,
+            params=tool_params if tool_params else []
+        )
+
+        return SuiteRunner(self.java, randoop.generate(), self.suite_env)
+
+    def evosuite_generate(self, tool_params=None):
+        evosuite = Evosuite(
+            java=self.java,
+            classpath=self.suite_env.classes_dir,
+            tests_src=self.suite_env.tests_src,
+            sut_class=self.suite_env.sut_class,
+            params=tool_params if tool_params else []
+        )
+
+        return SuiteRunner(self.java, evosuite.generate(), self.suite_env)
+
+class SuiteRunner():
+
+    def __init__(self, java, suite, suite_env):
+        self.java = java
+        self.suite = suite
+        self.suite_env = suite_env
+        self.junit = JUnit(java=self.java, classpath=suite_env.classes_dir)
+
+    def run_with_mutant(self, mutant):
+        return self.junit.run_with_mutant(self.suite, self.suite_env.sut_class, 
+                                          mutant)
+
+    def get_tool_name(self):
+        return self.suite.tool_name

nimrod/tools/evosuite.py

@@ -75,7 +75,8 @@ def generate_differential(self, mutant_classpath, make_dir=True):
 
         return Suite(suite_name=self.suite_name, suite_dir=self.suite_dir,
                      suite_classes_dir=self.suite_classes_dir,
-                     test_classes=self._test_classes())
+                     test_classes=self._test_classes(),
+                     tool_name=self._get_tool_name())
 
     # Argument for using only specific methods to test
     # Eg.: -Dtarget_method_list="append(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;"

nimrod/tools/java.py

@@ -41,7 +41,8 @@ def javap(self):
 
     @property
     def java(self):
-        return os.path.join(self.java_home, os.sep.join(['jre', 'bin', 'java']))
+        return os.path.join(self.java_home,
+                            os.sep.join(['jre', 'bin', 'java']))
 
     def _version_javac(self):
         return self.simple_exec_javac('-version')
@@ -101,11 +102,36 @@ def get_env(self, variables=None):
         return env
 
     def compile_all(self, classpath, directory):
-        if os.path.exists(directory):
-            java_files = get_java_files(directory)
-            for java_file in java_files:
-                class_file = os.path.join(directory,
-                                          java_file.replace('.java', '.class'))
-                if not os.path.exists(class_file):
-                    self.exec_javac(java_file, directory, None, None,
-                                    '-classpath', classpath)
+        args = '-classpath', classpath, '-d', directory
+        self.exec_javac_dir(directory, directory, self.get_env(), None,
+                            True, *args)
+
+    def exec_javac_dir(self, directory, cwd, env, timeout, skip_exists, *args):
+        java_files = get_java_files(directory)
+
+        if skip_exists:
+            java_files = self.get_not_compiled_files(java_files, *args)
+
+        self.exec_java_all(java_files, cwd, env, timeout, *args)
+
+    @staticmethod
+    def get_not_compiled_files(java_files, *args):
+        not_compiled = []
+        dest_dir = Java.get_dest_dir(args)
+        for f in java_files:
+            class_file = os.path.join(dest_dir, f.replace('.java', '.class'))
+            if not os.path.exists(class_file):
+                not_compiled.append(f)
+
+        return not_compiled
+
+    def exec_java_all(self, java_files, cwd, env, timeout, *args):
+        for java_file in java_files:
+            self.exec_javac(java_file, cwd, env, timeout, *args)
+
+    @staticmethod
+    def get_dest_dir(args):
+        try:
+            return args[args.index('-d') + 1]
+        except (ValueError, IndexError):
+            return ''

nimrod/tools/mujava.py

@@ -48,7 +48,8 @@ def _log_to_mutant(self, log):
     def compile_mutants(self, classpath, mutants):
         print("Compiling mutants...")
         for mutant in mutants:
-            self.java.compile_all(classpath, mutant.dir)
+            if os.path.exists(mutant.dir):
+                self.java.compile_all(classpath, mutant.dir)
 
     @staticmethod
     def _get_line_number(number, operator):