Compare llvm IR with llvm-diff

binutils.py

@@ -2,6 +2,7 @@
 import subprocess
 import os
 import sys
+import re
 
 
 def dump_asm(filename: str, objdump):
@@ -17,6 +18,35 @@ def dump_bc(base, dis, count, data):
     subprocess.run([dis, "seg{}.bc".format(count)], cwd=base)
 
 
+def diff_ir(out_dir, base1, base2, llvm_diff):
+    if not (os.path.exists(base1) and os.path.exists(base2)):
+        return
+
+    os.makedirs(out_dir, exist_ok=True)
+    diff_file = out_dir+"/diff"
+    with open(diff_file, 'w') as diff:
+        subprocess.run(['diff', '-x', '"*.bc"', '-q',
+                       '-r', base1, base2], stdout=diff)
+
+    pattern = re.compile(r'Files (.+) and (.+) differ')
+
+    diff_count = 0
+    with open(diff_file) as diff:
+        for line in diff.readlines():
+            matched = re.match(pattern, line.removesuffix('\n'))
+            if matched is not None:
+                lhs = matched.group(1)
+                rhs = matched.group(2)
+                irdiff_file = out_dir + "/irdiff"+str(diff_count)
+
+                with open(irdiff_file, 'w') as irdiff:
+                    irdiff.write(line)
+                    irdiff.flush()
+                    subprocess.run([llvm_diff, lhs, rhs], stderr=irdiff)
+
+                diff_count += 1
+
+
 def extract_bc(filename: str, objcopy, dis):
     bc_header = bytes([0x42, 0x43, 0xc0, 0xde])
     basedir = filename+"_bc"
@@ -52,19 +82,22 @@ def check_access(bin):
 
 
 if __name__ == '__main__':
-    if len(sys.argv) < 3:
-        print('Usage: binutils.py <llvm-bin-path> binaries...')
+    if len(sys.argv) != 4:
+        print('Usage: binutils.py <llvm-bin-path> binary1 binary2')
         exit(1)
 
     llvm_path = os.path.abspath(sys.argv[1]).removesuffix('/')
     llvm_dis = llvm_path + '/llvm-dis'
-    llvm_objdump = llvm_path+"/llvm-objdump"
-    llvm_objcopy = llvm_path+"/llvm-objcopy"
+    llvm_objdump = llvm_path + "/llvm-objdump"
+    llvm_objcopy = llvm_path + "/llvm-objcopy"
+    llvm_diff = llvm_path + "/llvm-diff"
 
-    if not (check_access(llvm_dis) and check_access(llvm_objdump) and check_access(llvm_objcopy)):
+    if not (check_access(llvm_dis) and check_access(llvm_objdump) and check_access(llvm_objcopy) and check_access(llvm_diff)):
         print('Error: invalid llvm binaries path')
         exit(1)
 
     for bin in sys.argv[2:]:
         dump_asm(bin, llvm_objdump)
         extract_bc(bin, llvm_objcopy, llvm_dis)
+
+    diff_ir("irdiff", sys.argv[2]+"_bc", sys.argv[3]+"_bc", llvm_diff)

report-generate.py

@@ -30,6 +30,8 @@ def parse(path):
         for test in tests:
             name = test['name']
             metrics = test['metrics']
+            if 'size' not in metrics:
+                continue
             size = metrics['size..text']
             if size > 0:
                 res[name] = (metrics['hash'], size)
@@ -96,6 +98,8 @@ def dump_diff(report, lhs_data, rhs_data):
             strip_name(name), lhs_hash, rhs_hash, lhs_size, rhs_size, rhs_size/lhs_size))
         copy_binary(binaries_src+lhs_hash, binaries_dst+lhs_hash)
         copy_binary(binaries_src+rhs_hash, binaries_dst+rhs_hash)
+        binutils.diff_ir(binaries_dst+'irdiff-{}-{}'.format(lhs_hash, rhs_hash), binaries_dst +
+                         lhs_hash, binaries_dst+rhs_hash, os.path.abspath(llvm_path+"llvm-diff"))
 
     if len(lhs_list) > 0:
         gmean_lhs = statistics.geometric_mean(lhs_list)