Merge pull request #2272 from dotty-staging/idempotency-checks

Fix #2185: Add bytecode idempotency checks.

Author: DarkDimius

compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -6,25 +6,13 @@ package tasty
 import TastyFormat._
 import collection.mutable
 import TastyBuffer._
-import java.util.UUID
 import core.Symbols.Symbol
 import ast.tpd
 import Decorators._
 
 class TastyPickler {
 
   private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)]
-  val uuid = UUID.randomUUID()
-
-  private val headerBuffer = {
-    val buf = new TastyBuffer(24)
-    for (ch <- header) buf.writeByte(ch.toByte)
-    buf.writeNat(MajorVersion)
-    buf.writeNat(MinorVersion)
-    buf.writeUncompressedLong(uuid.getMostSignificantBits)
-    buf.writeUncompressedLong(uuid.getLeastSignificantBits)
-    buf
-  }
 
   val nameBuffer = new NameBuffer
 
@@ -36,6 +24,20 @@ class TastyPickler {
       buf.assemble()
       buf.length + natSize(buf.length)
     }
+
+    val uuidLow: Long = pjwHash64(nameBuffer.bytes)
+    val uuidHi: Long = sections.iterator.map(x => pjwHash64(x._2.bytes)).fold(0L)(_ ^ _)
+
+    val headerBuffer = {
+      val buf = new TastyBuffer(header.length + 24)
+      for (ch <- header) buf.writeByte(ch.toByte)
+      buf.writeNat(MajorVersion)
+      buf.writeNat(MinorVersion)
+      buf.writeUncompressedLong(uuidLow)
+      buf.writeUncompressedLong(uuidHi)
+      buf
+    }
+
     val totalSize =
       headerBuffer.length +
       lengthWithLength(nameBuffer) + {
@@ -69,4 +71,23 @@ class TastyPickler {
   var addrOfSym: Symbol => Option[Addr] = (_ => None)
 
   val treePkl = new TreePickler(this)
+
+  /** Returns a non-cryptographic 64-bit hash of the array.
+   *
+   *  from https://en.wikipedia.org/wiki/PJW_hash_function#Implementation
+   */
+  private def pjwHash64(data: Array[Byte]): Long = {
+    var h = 0L
+    var high = 0L
+    var i = 0
+    while (i < data.length) {
+      h = (h << 4) + data(i)
+      high = h & 0xF0000000L
+      if (high != 0)
+        h ^= high >> 24
+      h &= ~high
+      i += 1
+    }
+    h
+  }
 }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -207,30 +207,6 @@ class CompilationTests extends ParallelTesting {
       compileDir("../library/src",
         allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))
 
-    def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
-      paths.iterator().asScala
-        .filter(path =>
-          (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
-            && !excludedFiles.contains(path.getFileName.toString))
-        .map(_.toString).toList
-
-    val compilerDir = Paths.get("../compiler/src")
-    val compilerSources = sources(Files.walk(compilerDir))
-
-    val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
-    val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
-
-    // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
-    val backendExcluded =
-      List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
-    val backendJvmExcluded =
-      List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")
-
-    val backendSources =
-      sources(Files.list(backendDir), excludedFiles = backendExcluded)
-    val backendJvmSources =
-      sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
-
     def dotty1 = {
       compileList(
         "dotty1",
@@ -261,6 +237,117 @@ class CompilationTests extends ParallelTesting {
       } :: Nil
     }.map(_.checkCompile()).foreach(_.delete())
   }
+
+  @Test def bytecodeIdemporency: Unit = {
+    var failed = 0
+    var total = 0
+    val blacklisted = Set(
+      // Bridges on collections in different order. Second one in scala2 order.
+      "pos/Map/scala/collection/immutable/Map",
+      "pos/Map/scala/collection/immutable/AbstractMap",
+      "pos/t1203a/NodeSeq",
+      "pos/i2345/Whatever"
+    )
+    def checkIdempotency(): Unit = {
+      val groupedBytecodeFiles: List[(Path, Path, Path, Path)] = {
+        val bytecodeFiles = {
+          def bytecodeFiles(paths: JStream[Path]): List[Path] = {
+            def isBytecode(file: String) = file.endsWith(".class") || file.endsWith(".tasty")
+            paths.iterator.asScala.filter(path => isBytecode(path.toString)).toList
+          }
+          val compilerDir1 = Paths.get("../out/idempotency1")
+          val compilerDir2 = Paths.get("../out/idempotency2")
+          bytecodeFiles(Files.walk(compilerDir1)) ++ bytecodeFiles(Files.walk(compilerDir2))
+        }
+        val groups = bytecodeFiles.groupBy(f => f.toString.substring("../out/idempotencyN/".length, f.toString.length - 6))
+        groups.filterNot(x => blacklisted(x._1)).valuesIterator.flatMap { g =>
+          def pred(f: Path, i: Int, isTasty: Boolean) =
+            f.toString.contains("idempotency" + i) && f.toString.endsWith(if (isTasty) ".tasty" else ".class")
+          val class1 = g.find(f => pred(f, 1, isTasty = false))
+          val class2 = g.find(f => pred(f, 2, isTasty = false))
+          val tasty1 = g.find(f => pred(f, 1, isTasty = true))
+          val tasty2 = g.find(f => pred(f, 2, isTasty = true))
+          assert(class1.isDefined, "Could not find class in idempotency1 for " + class2)
+          assert(class2.isDefined, "Could not find class in idempotency2 for " + class1)
+          if (tasty1.isEmpty || tasty2.isEmpty) Nil
+          else List(Tuple4(class1.get, tasty1.get, class2.get, tasty2.get))
+        }.toList
+      }
+
+      for ((class1, tasty1, class2, tasty2) <- groupedBytecodeFiles) {
+        total += 1
+        val bytes1 = Files.readAllBytes(class1)
+        val bytes2 = Files.readAllBytes(class2)
+        if (!java.util.Arrays.equals(bytes1, bytes2)) {
+          failed += 1
+          val tastyBytes1 = Files.readAllBytes(tasty1)
+          val tastyBytes2 = Files.readAllBytes(tasty2)
+          if (java.util.Arrays.equals(tastyBytes1, tastyBytes2))
+            println(s"Idempotency test failed between $class1 and $class1 (same tasty)")
+          else
+            println(s"Idempotency test failed between $tasty1 and $tasty2")
+          /* Dump bytes to console, could be useful if issue only appears in CI.
+           * Create the .class locally with Files.write(path, Array[Byte](...)) with the printed array
+           */
+          // println(bytes1.mkString("Array[Byte](", ",", ")"))
+          // println(bytes2.mkString("Array[Byte](", ",", ")"))
+        }
+      }
+    }
+
+    val opt = defaultOptions.and("-YemitTasty")
+
+    def idempotency1() = {
+      compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) +
+      compileFilesInDir("../tests/pos", opt)
+    }
+    def idempotency2() = {
+      compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) +
+      compileFilesInDir("../tests/pos", opt)
+    }
+
+    val tests = (idempotency1() + idempotency2()).keepOutput.checkCompile()
+
+    assert(new java.io.File("../out/idempotency1/").exists)
+    assert(new java.io.File("../out/idempotency2/").exists)
+
+    val t0 = System.currentTimeMillis()
+    checkIdempotency()
+    println(s"checked bytecode idempotency (${(System.currentTimeMillis() - t0) / 1000.0} sec)")
+
+    tests.delete()
+
+    assert(failed == 0, s"Failed $failed idempotency checks (out of $total)")
+  }
+
+
+  private val (compilerSources, backendSources, backendJvmSources) = {
+    def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
+      paths.iterator().asScala
+        .filter(path =>
+          (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
+            && !excludedFiles.contains(path.getFileName.toString))
+        .map(_.toString).toList
+
+    val compilerDir = Paths.get("../compiler/src")
+    val compilerSources0 = sources(Files.walk(compilerDir))
+
+    val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
+    val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
+
+    // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
+    val backendExcluded =
+      List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
+    val backendJvmExcluded =
+      List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")
+
+    val backendSources0 =
+      sources(Files.list(backendDir), excludedFiles = backendExcluded)
+    val backendJvmSources0 =
+      sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
+
+    (compilerSources0, backendSources0, backendJvmSources0)
+  }
 }
 
 object CompilationTests {