improve testing infrastructure and fix small bugs

Author: poechsel

semanticdb/input/src/main/scala/example/Types.scala

@@ -117,4 +117,4 @@ object TypTest {
     final val javaEnum = java.nio.file.LinkOption.NOFOLLOW_LINKS
     final val clazzOf = classOf[Option[Int]]
   }
-}
+}

semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala

@@ -281,7 +281,7 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
         }
 
         def isUseless(implicit ctx: Context): Boolean = {
-          symbol == NoSymbol ||
+          (symbol.name == "<none>" || symbol == NoSymbol) ||
           symbol.isReservedName ||
           symbol.isAnonymousInit ||
           symbol.isDefaultGetter ||
@@ -296,6 +296,9 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
           symbol.isSyntheticCaseAccessor ||
           symbol.isRefinementClass ||
           symbol.isSyntheticJavaModule
+          // isSyntheticJavaModule disable the symbol Class in
+          // Class.forName(???) to be recorded as Class is considered to
+          // be a class in dotty, not a typed.
         }
         def isUseful(implicit ctx: Context): Boolean = !symbol.isUseless
         def isUselessOccurrence(implicit ctx: Context): Boolean = {
@@ -427,8 +430,6 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
                        isMutableAssignement:Boolean = false): Unit = {
         if (symbol.name == "<none>") return
 
-
-          println("===> ", symbol, symbol.flags)
         val symbolName = if (isMutableAssignement) symbol.trueName + "_=" else symbol.trueName
         val (symbol_path, is_global) = posToRange(symbol.pos) match {
           case Some(keyRange)
@@ -456,12 +457,8 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
         // dotty will generate a ValDef for the x, but the x will also
         // be present in the constructor, thus making a double definition
         if (symbolPathsMap.contains(key)) return
-        //if (is_global) {
-          symbolPathsMap += key
-        //}
-        println(symbol_path,
-                range,
-                symbol.flags)
+
+        symbolPathsMap += key
         occurrences =
           occurrences :+
             s.SymbolOccurrence(
@@ -490,15 +487,15 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
 
       val reservedFunctions: List[String] = Nil
       def addOccurenceTree(tree: Tree,
-                           type_symbol: s.SymbolOccurrence.Role,
+                           typeSymbol: s.SymbolOccurrence.Role,
                            range: s.Range,
                            force_add: Boolean = false,
                            isMutableAssignement: Boolean = false): Unit = {
         if (tree.symbol.isUseful &&
-          isMutableSetterExplicit(tree.symbol, type_symbol) &&
+          isMutableSetterExplicit(tree.symbol, typeSymbol) &&
           (tree.isUserCreated ||
-            (force_add && !(!tree.isUserCreated && iterateParent(tree.symbol) == "java/lang/Object#`<init>`().")))) {
-          addOccurence(tree.symbol, type_symbol, range, isMutableAssignement)
+            (force_add /*&& !(!tree.isUserCreated && iterateParent(tree.symbol) == "java/lang/Object#`<init>`().")*/))) {
+          addOccurence(tree.symbol, typeSymbol, range, isMutableAssignement)
         }
       }
       def addOccurenceTypeTree(typetree: TypeTree,
@@ -761,7 +758,6 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
         val start = Map[String, s.Range]()
         params.foldLeft(start)((old, statements) => {
           statements.foldLeft(old)((old, cval) => {
-            println(cval)
             old + (cval.name -> range(cval, cval.symbol.pos, cval.symbol.trueName))
           })
         }
@@ -833,9 +829,7 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
             forceAddBecauseParents = !(tree.symbol.flags.is(Flags.Case))
             parents.foreach(_ match {
               case IsTypeTree(t) => traverseTypeTree(t)
-              case IsTerm(t) => {
-                traverseTree(t)
-              }
+              case IsTerm(t) => traverseTree(t)
             })
             forceAddBecauseParents = false
             selfopt match {
@@ -870,31 +864,20 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
               }
               case _ =>
             }
-            if (!tree.symbol.flags.is(Flags.Case)) {
+
             classStacks = tree.symbol :: classStacks
 
-            println(statements)
-            println("")
             val paramsPosMapping = generateParamsPosMapping(constr)
-            println(paramsPosMapping)
-            println("")
 
             statements.foreach(statement => {
               if (statement.symbol.flags.is(Flags.ParamAccessor)) {
                 if (paramsPosMapping.contains(statement.symbol.name)) {
-                  println("parameter "+statement)
-                  addOccurence(statement.symbol, s.SymbolOccurrence.Role.DEFINITION, paramsPosMapping(statement.symbol.name))
+                  addOccurenceTree(statement, s.SymbolOccurrence.Role.DEFINITION, paramsPosMapping(statement.symbol.name))
                 }
-                //traverseTree(statement)
               } else if (!statement.symbol.flags.is(Flags.Param)) {
-                println(statement.symbol, statement.symbol.flags)
                 traverseTree(statement)
               }
             })
-
-            classStacks = classStacks.tail
-            }
-
           }
 
           case DefDef("<init>", typeparams, valparams, type_, statements) if fittedInitClassRange != None => {
@@ -914,13 +897,12 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
 
           case Term.Assign(lhs, rhs) => {
             isAssignedTerm = true
-            super.traverseTree(tree)
+            traverseTree(lhs)
             isAssignedTerm = false
+            traverseTree(rhs)
           }
 
           case IsDefinition(cdef) => {
-            println("definition " + cdef.symbol + " " + cdef.symbol.flags)
-            println(cdef.symbol.protectedWithin, cdef.symbol.privateWithin)
             if (cdef.symbol.flags.is(Flags.Protected)) {
               cdef.symbol.protectedWithin match {
                 case Some(within) => {
@@ -939,7 +921,6 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
             } else {
               cdef.symbol.privateWithin match {
                 case Some(within) => {
-                  println("YES")
                   val startColumn = cdef.pos.startColumn + "private[".length
                   addOccurence(
                     within.typeSymbol,
@@ -978,10 +959,24 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
             super.traverseTree(cdef)
           }
 
-          case Term.This(Some(_)) => {
+          case Term.This(Some(id)) => {
+            /* We've got two options here:
+            - either the this is explicit: eg C.this.XXX. In this case, the position is [C.this], but
+              we want to put the symbol on the C, so around id
+            - either it is not explicit (eg a.foo). We want to put the symbol only around the a.
+            Distinguishing between the two is easy. If the sourcecode between [pos.start; pos.end] ends
+            with a 'this', then we're in the first case, otherwise the second
+            */
+            var rangeThis = posToRange(tree.pos).get
+            if (sourceCode.substring(tree.pos.start, tree.pos.end).endsWith("this")) {
+              rangeThis = range(tree, tree.pos, tree.symbol.trueName)
+            }
+            /*range = s.Range(tree.pos.startLine, tree.pos.startColumn,
+            tree.pos.endLine,
+            )*/
             addOccurenceTree(tree,
                              s.SymbolOccurrence.Role.REFERENCE,
-                             posToRange(tree.pos).get)
+                             rangeThis)
           }
 
           case Term.Super(_, Some(id)) =>
@@ -1001,14 +996,15 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
                 range = s.Range(tree.pos.startLine, tree.pos.start - 5, tree.pos.startLine, tree.pos.start - 1)
                 shouldForceAdd = true
               } else {
+                range = s.Range(tree.pos.endLine, tree.pos.endColumn, tree.pos.endLine, tree.pos.endColumn)
                 shouldForceAdd = qualifier.isUserCreated
               }
             }
             val temp = isAssignedTerm
             isAssignedTerm = false
             super.traverseTree(tree)
             isAssignedTerm = temp
-            addOccurenceTree(tree, s.SymbolOccurrence.Role.REFERENCE, range, shouldForceAdd, isAssignedTerm && tree.symbol.flags.is(Flags.Mutable))
+            addOccurenceTree(tree, s.SymbolOccurrence.Role.REFERENCE, range, shouldForceAdd, isAssignedTerm && tree.symbol.flags.is(Flags.Mutable) && !tree.symbol.flags.is(Flags.PrivateLocal))
           }
 
           case Term.Ident(name) => {
@@ -1062,9 +1058,6 @@ class SemanticdbConsumer(sourceFile: java.nio.file.Path) extends TastyConsumer {
       }
 
     }
-    println("{--------------------------------------}")
-    println(root)
-    println("{--------------------------------------}")
 
     Traverser.traverseTree(root)(reflect.rootContext)
   }

semanticdb/test/dotty/semanticdb/Tests.scala

@@ -14,8 +14,74 @@ import scala.tasty.Reflection
 import scala.tasty.file.TastyConsumer
 import java.lang.reflect.InvocationTargetException
 
+import dotty.semanticdb.Scala._
+
+
 class Tests {
 
+  def min(a: Int, b: Int) : Int = if (a > b) b else a
+  def max(a: Int, b: Int) : Int = if (a > b) a else b
+  def abs(a: Int, b: Int) : Int = max(a, b) - min(a, b)
+  def distance(r1 : s.Range, offsets : Array[Int])(r2 : s.Range) : Int = {
+    val s1 = offsets(max(r1.startLine, 0)) + r1.startCharacter
+    val s2 = offsets(max(r2.startLine, 0)) + r2.startCharacter
+    val e1 = offsets(max(r1.endLine, 0)) + r1.endCharacter
+    val e2 = offsets(max(r2.endLine, 0)) + r2.endCharacter
+    max(abs(s1, s2), abs(e1, e2))
+  }
+
+  def compareOccurences(tastyOccurences : Seq[s.SymbolOccurrence],
+    scalaOccurences : Seq[s.SymbolOccurrence],
+    sourceCode : String)
+    : Boolean= {
+      val lineToByte = sourceCode.split("\n").scanLeft(0)((o, l) => o + l.length + 1)
+      val symbols = tastyOccurences.groupBy(_.symbol)
+      val localTastyToScala = HashMap[String, String]()
+      val localScalaToTasty = HashMap[String, String]()
+      val translator = HashMap[(s.Range, String), s.SymbolOccurrence]()
+
+      // from is in tasty space, to in scala space
+      def checkIfTranslatableSymbol(from : String, to : String) : Boolean = {
+        if (from.isLocal != to.isLocal) {
+          false
+        } else {
+          if (from.isLocal) {
+            if(localTastyToScala.getOrElse(from, to) == to &&
+              localScalaToTasty.getOrElse(to, from) == from) {
+            localTastyToScala += (from -> to)
+            localScalaToTasty += (to -> from)
+              true
+            } else {
+              false
+            }
+          } else {
+            true
+          }
+        }
+      }
+
+      if (tastyOccurences.length != scalaOccurences.length) {
+        false
+      } else {
+        scalaOccurences.forall(occurence => {
+          if (symbols.contains(localScalaToTasty.getOrElse(occurence.symbol, occurence.symbol))) {
+            val siblings = symbols(occurence.symbol)
+            val nearest = siblings.minBy((c : s.SymbolOccurrence) => distance(occurence.range.get, lineToByte)(c.range.get))
+            if (!checkIfTranslatableSymbol(nearest.symbol, occurence.symbol) ||
+               translator.contains((nearest.range.get, nearest.symbol)) ||
+               distance(occurence.range.get, lineToByte)(nearest.range.get) > 5) {
+              false
+            } else {
+              translator += ((nearest.range.get, nearest.symbol) -> occurence)
+              true
+            }
+          } else {
+            false
+          }
+        })
+      }
+  }
+
   // TODO: update scala-0.13 on version change (or resolve automatically)
   final def tastyClassDirectory =
     Paths.get("out/bootstrap/dotty-semanticdb/scala-0.12/test-classes/")
@@ -34,9 +100,6 @@ class Tests {
   /** Returns the SemanticDB for this Scala source file. */
   def getTastySemanticdb(classPath: Path, scalaFile: Path) : s.TextDocument = {
     val classNames = Utils.getClassNames(classPath, scalaFile, "example/")
-    println(classPath)
-    println(classNames)
-    println(scalaFile)
     val sdbconsumer = new SemanticdbConsumer(scalaFile)
 
     val _ = ConsumeTasty(classPath.toString, classNames, sdbconsumer)
@@ -48,11 +111,10 @@ class Tests {
     val path = sourceDirectory.resolve(filename)
     val scalac = getScalacSemanticdb(path)
     val tasty = getTastySemanticdb(tastyClassDirectory, path)
-    println(tasty)
     val obtained = Semanticdbs.printTextDocument(tasty)
     val expected = Semanticdbs.printTextDocument(scalac)
-    print("X=>",scalac.occurrences)
-    assertNoDiff(obtained, expected)
+    if (!compareOccurences(tasty.occurrences, scalac.occurrences, scalac.text))
+      assertNoDiff(obtained, expected)
   }
 
   /** Fails the test with a pretty diff if there obtained is not the same as expected */
@@ -83,7 +145,7 @@ class Tests {
   }
 
 
-  /*@Test def testAccess(): Unit = checkFile("example/Access.scala")
+  @Test def testAccess(): Unit = checkFile("example/Access.scala")
   @Test def testAdvanced(): Unit = checkFile("example/Advanced.scala")
   @Test def testAnonymous(): Unit = checkFile("example/Anonymous.scala")
   @Test def testClasses(): Unit = checkFile("example/Classes.scala")
@@ -119,12 +181,5 @@ class Tests {
   @Test def testSynthetic(): Unit = checkFile("example/Synthetic.scala")
   @Test def testBinaryOp(): Unit = checkFile("example/BinaryOp.scala")
   @Test def testDottyPredef(): Unit = checkFile("example/DottyPredef.scala")
-  */
-  //@Test def testTypesAnnotations() : Unit = checkFile("example/TypesAnnotations.scala") // Crash, has to deal with init symbols
-  //@Test def testNew(): Unit = checkFile("example/New.scala")
-  //@Test def testClasses(): Unit = checkFile("example/Classes.scala")
 
-  //@Test def testSemanticDoc(): Unit = checkFile("example/SemanticDoc.scala")
-  @Test def testAccess(): Unit = checkFile("example/Access.scala")
-  //@Test def testDependantModule(): Unit = checkFile("example/DependantModule.scala")
 }