Merge pull request github#5664 from tamasvajk/feature/stub-generation

C#: Stub generation

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs

@@ -289,7 +289,7 @@ private void AnalyseProject(FileInfo project)
 
             try
             {
-                var csProj = new CsProjFile(project);
+                var csProj = new Extraction.CSharp.CsProjFile(project);
 
                 foreach (var @ref in csProj.References)
                 {

csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs

@@ -47,7 +47,10 @@ protected override void ExtractInitializers(TextWriter trapFile)
                     var baseType = Symbol.ContainingType.BaseType;
                     if (baseType is null)
                     {
-                        Context.ModelError(Symbol, "Unable to resolve base type in implicit constructor initializer");
+                        if (Symbol.ContainingType.SpecialType != SpecialType.System_Object)
+                        {
+                            Context.ModelError(Symbol, "Unable to resolve base type in implicit constructor initializer");
+                        }
                         return;
                     }
 

…traction.CSharp.Standalone/CsProjFile.cs …xtraction.CSharp/Extractor/CsProjFile.cscsharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs renamed to csharp/extractor/Semmle.Extraction.CSharp/Extractor/CsProjFile.cs csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs renamed to csharp/extractor/Semmle.Extraction.CSharp/Extractor/CsProjFile.cs

@@ -4,12 +4,12 @@
 using System.Linq;
 using System.Xml;
 
-namespace Semmle.BuildAnalyser
+namespace Semmle.Extraction.CSharp
 {
     /// <summary>
     /// Represents a .csproj file and reads information from it.
     /// </summary>
-    internal class CsProjFile
+    public class CsProjFile
     {
         private string Filename { get; }
 
@@ -38,14 +38,14 @@ public CsProjFile(FileInfo filename)
                 // unrecognised content or is the wrong version.
                 // This currently always fails on Linux because
                 // Microsoft.Build is not cross platform.
-                (csFiles, references) = ReadMsBuildProject(filename);
+                (csFiles, references, projectReferences) = ReadMsBuildProject(filename);
             }
             catch  // lgtm[cs/catch-of-all-exceptions]
             {
                 // There was some reason why the project couldn't be loaded.
                 // Fall back to reading the Xml document directly.
                 // This method however doesn't handle variable expansion.
-                (csFiles, references) = ReadProjectFileAsXml(filename, Directory);
+                (csFiles, references, projectReferences) = ReadProjectFileAsXml(filename, Directory);
             }
         }
 
@@ -55,7 +55,7 @@ public CsProjFile(FileInfo filename)
         /// and there seems to be no way to make it succeed. Fails on Linux.
         /// </summary>
         /// <param name="filename">The file to read.</param>
-        private static (string[] csFiles, string[] references) ReadMsBuildProject(FileInfo filename)
+        private static (string[] csFiles, string[] references, string[] projectReferences) ReadMsBuildProject(FileInfo filename)
         {
             var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName);
 
@@ -64,13 +64,18 @@ private static (string[] csFiles, string[] references) ReadMsBuildProject(FileIn
                 .Select(item => item.EvaluatedInclude)
                 .ToArray();
 
+            var projectReferences = msbuildProject.Items
+                .Where(item => item.ItemType == "ProjectReference")
+                .Select(item => item.EvaluatedInclude)
+                .ToArray();
+
             var csFiles = msbuildProject.Items
                 .Where(item => item.ItemType == "Compile")
                 .Select(item => item.GetMetadataValue("FullPath"))
                 .Where(fn => fn.EndsWith(".cs"))
                 .ToArray();
 
-            return (csFiles, references);
+            return (csFiles, references, projectReferences);
         }
 
         /// <summary>
@@ -79,7 +84,7 @@ private static (string[] csFiles, string[] references) ReadMsBuildProject(FileIn
         /// fallback if ReadMsBuildProject() fails.
         /// </summary>
         /// <param name="fileName">The .csproj file.</param>
-        private static (string[] csFiles, string[] references) ReadProjectFileAsXml(FileInfo fileName, string directoryName)
+        private static (string[] csFiles, string[] references, string[] projectReferences) ReadProjectFileAsXml(FileInfo fileName, string directoryName)
         {
             var projFile = new XmlDocument();
             var mgr = new XmlNamespaceManager(projFile.NameTable);
@@ -109,8 +114,16 @@ private static (string[] csFiles, string[] references) ReadProjectFileAsXml(File
 
                 var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", SearchOption.AllDirectories);
 
+                var projectReferences = root
+                    .SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr)
+                    ?.NodeList()
+                    .Select(node => node.Value)
+                    .Select(csproj => GetFullPath(csproj, projDir))
+                    .Where(s => s is not null)
+                    ?? Enumerable.Empty<string>();
+
 #nullable disable warnings
-                return (explicitCsFiles.Concat(additionalCsFiles).ToArray(), Array.Empty<string>());
+                return (explicitCsFiles.Concat(additionalCsFiles).ToArray(), Array.Empty<string>(), projectReferences.ToArray());
 #nullable restore warnings
             }
 
@@ -135,7 +148,7 @@ private static (string[] csFiles, string[] references) ReadProjectFileAsXml(File
                 .ToArray();
 
 #nullable disable warnings
-            return (csFiles, references);
+            return (csFiles, references, Array.Empty<string>());
 #nullable restore warnings
         }
 
@@ -150,13 +163,19 @@ private static (string[] csFiles, string[] references) ReadProjectFileAsXml(File
         }
 
         private readonly string[] references;
+        private readonly string[] projectReferences;
         private readonly string[] csFiles;
 
         /// <summary>
         /// The list of references as a list of assembly IDs.
         /// </summary>
         public IEnumerable<string> References => references;
 
+        /// <summary>
+        /// The list of project references in full path format.
+        /// </summary>
+        public IEnumerable<string> ProjectReferences => projectReferences;
+
         /// <summary>
         /// The list of C# source files in full path format.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -107,6 +107,11 @@ public static ExitCode Run(string[] args)
 
             try
             {
+                if (options.ProjectsToLoad.Any())
+                {
+                    AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
+                }
+
                 var compilerVersion = new CompilerVersion(options);
 
                 if (compilerVersion.SkipExtraction)
@@ -146,6 +151,41 @@ public static ExitCode Run(string[] args)
             }
         }
 
+        private static void AddSourceFilesFromProjects(IEnumerable<string> projectsToLoad, IList<string> compilerArguments, ILogger logger)
+        {
+            logger.Log(Severity.Info, "  Loading referenced projects.");
+            var projects = new Queue<string>(projectsToLoad);
+            var processed = new HashSet<string>();
+            while (projects.Count > 0)
+            {
+                var project = projects.Dequeue();
+                var fi = new FileInfo(project);
+                if (processed.Contains(fi.FullName))
+                {
+                    continue;
+                }
+
+                processed.Add(fi.FullName);
+                logger.Log(Severity.Info, "  Processing referenced project: " + fi.FullName);
+
+                var csProj = new CsProjFile(fi);
+
+                foreach (var cs in csProj.Sources)
+                {
+                    if (cs.Contains("/obj/"))
+                    {
+                        continue;
+                    }
+                    compilerArguments.Add(cs);
+                }
+
+                foreach (var pr in csProj.ProjectReferences)
+                {
+                    projects.Enqueue(pr);
+                }
+            }
+        }
+
         /// <summary>
         /// Gets the complete list of locations to locate references.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -16,6 +16,12 @@ public sealed class Options : CommonOptions
         /// </summary>
         public string? Framework { get; set; }
 
+        /// <summary>
+        /// Project files whose source files should be added to the compilation.
+        /// Only used in tests.
+        /// </summary>
+        public IList<string> ProjectsToLoad { get; } = new List<string>();
+
         /// <summary>
         /// All other arguments passed to the compilation.
         /// </summary>
@@ -68,6 +74,9 @@ public override bool HandleOption(string key, string value)
                 case "framework":
                     Framework = value;
                     return true;
+                case "load-sources-from-project":
+                    ProjectsToLoad.Add(value);
+                    return true;
                 default:
                     return base.HandleOption(key, value);
             }

csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj

@@ -23,6 +23,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.9.0" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
   </ItemGroup>
 
 </Project>

csharp/ql/src/Stubs/AllStubsFromReference.ql

@@ -0,0 +1,15 @@
+/**
+ * Tool to generate C# stubs from a qltest snapshot.
+ */
+
+import csharp
+import Stubs
+
+/** All public declarations from assemblies. */
+class AllExternalPublicDeclarations extends GeneratedDeclaration {
+  AllExternalPublicDeclarations() { this.fromLibrary() }
+}
+
+from Assembly a
+select a.getFullName(), a.getName(), a.getVersion().toString(), a.getFile().getAbsolutePath(),
+  generatedCode(a)

csharp/ql/src/Stubs/AllStubsFromSource.ql

@@ -0,0 +1,22 @@
+/**
+ * Tool to generate C# stubs from a qltest snapshot.
+ */
+
+import csharp
+import Stubs
+
+/** All public declarations from source. */
+class AllDeclarations extends GeneratedDeclaration {
+  AllDeclarations() { not this.fromLibrary() }
+}
+
+/** Exclude types from these standard assemblies. */
+private class DefaultLibs extends ExcludedAssembly {
+  DefaultLibs() {
+    this.getName() = "System.Private.CoreLib" or
+    this.getName() = "mscorlib" or
+    this.getName() = "System.Runtime"
+  }
+}
+
+select concat(generatedCode(_) + "\n\n")

csharp/ql/src/Stubs/MinimalStubsFromSource.ql

@@ -29,4 +29,13 @@ class UsedInSource extends GeneratedDeclaration {
   }
 }
 
-select generatedCode()
+/** Exclude types from these standard assemblies. */
+private class DefaultLibs extends ExcludedAssembly {
+  DefaultLibs() {
+    this.getName() = "System.Private.CoreLib" or
+    this.getName() = "mscorlib" or
+    this.getName() = "System.Runtime"
+  }
+}
+
+select concat(generatedCode(_) + "\n\n")