Go: merge with incoming path transformer when setting GOPATH

Author: nickrolfe

go/extractor/cli/go-autobuilder/BUILD.bazel

@@ -13,6 +13,7 @@ import (
 	"github.com/github/codeql-go/extractor/autobuilder"
 	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/project"
+	"github.com/github/codeql-go/extractor/srcarchive"
 	"github.com/github/codeql-go/extractor/toolchain"
 	"github.com/github/codeql-go/extractor/util"
 )
@@ -283,7 +284,7 @@ func createPathTransformerFile(newdir string) *os.File {
 }
 
 // Writes the path transformer file
-func writePathTransformerFile(pt *os.File, realSrc, root, newdir string) {
+func writePathTransformerFile(pt *os.File, realSrc, newdir string) {
 	_, err := pt.WriteString("#" + realSrc + "\n" + newdir + "//\n")
 	if err != nil {
 		log.Fatalf("Unable to write path transformer file: %s.", err.Error())
@@ -501,10 +502,6 @@ func installDependenciesAndBuild() {
 
 	srcdir := getSourceDir()
 
-	// we set `CODEQL_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency
-	os.Setenv("CODEQL_PATH_TRANSFORMER", "")
-	os.Setenv("SEMMLE_PATH_TRANSFORMER", "")
-
 	// determine how to install dependencies and whether a GOPATH needs to be set up before
 	// extraction
 	workspaces := project.GetWorkspaceInfo(true)
@@ -535,7 +532,21 @@ func installDependenciesAndBuild() {
 			pt := createPathTransformerFile(paths.newdir)
 			defer os.Remove(pt.Name())
 
-			writePathTransformerFile(pt, paths.realSrc, paths.root, paths.newdir)
+			// We're about to create out own path transformer, so that paths containing the
+			// temporary GOPATH point to the right location. However, if there was already an
+			// incoming path transformer, the right location will be what _it_ wanted to transform
+			// paths to.
+			existingPathTransformer, err := srcarchive.LoadProjectLayoutFromEnv()
+			if err != nil {
+				log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
+			}
+			var realSrc string
+			if existingPathTransformer == nil {
+				realSrc = paths.realSrc
+			} else {
+				realSrc = existingPathTransformer.To
+			}
+			writePathTransformerFile(pt, realSrc, paths.newdir)
 			setGopath(paths.root)
 		}
 	}

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"strings"
+
+	"github.com/github/codeql-go/extractor/util"
 )
 
 // ProjectLayout describes a very simple project layout rewriting paths starting
@@ -16,7 +18,7 @@ import (
 //	# to
 //	from//
 type ProjectLayout struct {
-	from, to string
+	From, To string
 }
 
 // normaliseSlashes adds an initial slash to `path` if there isn't one, and trims
@@ -28,6 +30,25 @@ func normaliseSlashes(path string) string {
 	return strings.TrimSuffix(path, "/")
 }
 
+// LoadProjectLayoutFromEnv loads a project layout from the file referenced by the
+// {CODEQL,SEMMLE}_PATH_TRANSFORMER environment variable. If neither env var is set, returns nil. If
+// the file cannot be read or does not have the right format, it returns an error.
+func LoadProjectLayoutFromEnv() (*ProjectLayout, error) {
+	pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER")
+	if pt == "" {
+		return nil, nil
+	}
+	ptf, err := os.Open(pt)
+	if err != nil {
+		return nil, err
+	}
+	projLayout, err := LoadProjectLayout(ptf)
+	if err != nil {
+		return nil, err
+	}
+	return projLayout, nil
+}
+
 // LoadProjectLayout loads a project layout from the given file, returning an error
 // if the file does not have the right format
 func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
@@ -41,7 +62,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
 	if !strings.HasPrefix(line, "#") {
 		return nil, fmt.Errorf("first line of project layout should start with #, but got %s", line)
 	}
-	res.to = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
+	res.To = normaliseSlashes(strings.TrimSpace(strings.TrimPrefix(line, "#")))
 
 	if !scanner.Scan() {
 		return nil, errors.New("empty section in project-layout file")
@@ -57,7 +78,7 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
 	if strings.HasPrefix(line, "-") || strings.Contains(line, "*") || strings.Contains(line, "//") {
 		return nil, errors.New("unsupported project-layout feature")
 	}
-	res.from = normaliseSlashes(line)
+	res.From = normaliseSlashes(line)
 
 	for scanner.Scan() {
 		if strings.TrimSpace(scanner.Text()) != "" {
@@ -71,11 +92,11 @@ func LoadProjectLayout(file *os.File) (*ProjectLayout, error) {
 // transformString transforms `str` as specified by the project layout: if it starts with the `from`
 // prefix, that prefix is relaced by `to`; otherwise the string is returned unchanged
 func (p *ProjectLayout) transformString(str string) string {
-	if str == p.from {
-		return p.to
+	if str == p.From {
+		return p.To
 	}
-	if strings.HasPrefix(str, p.from+"/") {
-		return p.to + "/" + str[len(p.from)+1:]
+	if strings.HasPrefix(str, p.From+"/") {
+		return p.To + "/" + str[len(p.From)+1:]
 	}
 	return str
 }

go/extractor/srcarchive/projectlayout.go

@@ -28,23 +28,45 @@ func mkProjectLayout(projectLayoutSource string, t *testing.T) (*ProjectLayout,
 	return LoadProjectLayout(pt)
 }
 
+func mkProjectLayoutFromEnv(projectLayoutSource string, t *testing.T) (*ProjectLayout, error) {
+	pt, err := os.CreateTemp("", "path-transformer-from-env")
+	if err != nil {
+		t.Fatalf("Unable to create temporary file for project layout: %s", err.Error())
+	}
+	defer os.Remove(pt.Name())
+	_, err = pt.WriteString(projectLayoutSource)
+	if err != nil {
+		t.Fatalf("Unable to write to temporary file for project layout: %s", err.Error())
+	}
+	err = pt.Close()
+	if err != nil {
+		t.Fatalf("Unable to close path transformer file: %s.", err.Error())
+	}
+
+	pt, err = os.Open(pt.Name())
+	if err != nil {
+		t.Fatalf("Unable to open path transformer file: %s.", err.Error())
+	}
+
+	os.Setenv("CODEQL_PATH_TRANSFORMER", pt.Name())
+	defer os.Unsetenv("CODEQL_PATH_TRANSFORMER")
+
+	return LoadProjectLayoutFromEnv()
+}
+
 func testTransformation(projectLayout *ProjectLayout, t *testing.T, path string, expected string) {
 	actual := projectLayout.Transform(path)
 	if actual != expected {
 		t.Errorf("Expected %s to be transformed to %s, but got %s", path, expected, actual)
 	}
 }
 
-func TestValidProjectLayout(t *testing.T) {
-	p, err := mkProjectLayout(`
+const validProjectLayoutSource = `
 # /opt/src
 /opt/src/root/src/org/repo//
-`, t)
-
-	if err != nil {
-		t.Fatalf("Error loading project layout: %s", err.Error())
-	}
+`
 
+func testTransformationsForValidProjectLayout(p *ProjectLayout, t *testing.T) {
 	testTransformation(p, t, "/opt/src/root/src/org/repo", "/opt/src")
 	testTransformation(p, t, "/opt/src/root/src/org/repo/", "/opt/src/")
 	testTransformation(p, t, "/opt/src/root/src/org/repo/main.go", "/opt/src/main.go")
@@ -53,6 +75,26 @@ func TestValidProjectLayout(t *testing.T) {
 	testTransformation(p, t, "opt/src/root/src/org/repo", "opt/src/root/src/org/repo")
 }
 
+func TestValidProjectLayout(t *testing.T) {
+	p, err := mkProjectLayout(validProjectLayoutSource, t)
+
+	if err != nil {
+		t.Fatalf("Error loading project layout: %s", err.Error())
+	}
+
+	testTransformationsForValidProjectLayout(p, t)
+}
+
+func TestValidProjectLayoutFromEnv(t *testing.T) {
+	p, err := mkProjectLayoutFromEnv(validProjectLayoutSource, t)
+
+	if err != nil {
+		t.Fatalf("Error loading project layout: %s", err.Error())
+	}
+
+	testTransformationsForValidProjectLayout(p, t)
+}
+
 func TestWindowsPaths(t *testing.T) {
 	p, err := mkProjectLayout(`
 # /c:/virtual

go/extractor/srcarchive/projectlayout_test.go

@@ -7,23 +7,16 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-
-	"github.com/github/codeql-go/extractor/util"
 )
 
 var pathTransformer *ProjectLayout
 
 func init() {
-	pt := util.Getenv("CODEQL_PATH_TRANSFORMER", "SEMMLE_PATH_TRANSFORMER")
-	if pt != "" {
-		ptf, err := os.Open(pt)
-		if err != nil {
-			log.Fatalf("Unable to open path transformer %s: %s.\n", pt, err.Error())
-		}
-		pathTransformer, err = LoadProjectLayout(ptf)
-		if err != nil {
-			log.Fatalf("Unable to initialize path transformer: %s.\n", err.Error())
-		}
+	pt, err := LoadProjectLayoutFromEnv()
+	if err == nil {
+		pathTransformer = pt
+	} else {
+		log.Fatalf("Unable to load path transformer: %s.\n", err.Error())
 	}
 }