Support private dependencies (#348)

Author: chrismwendt

Gopkg.lock

@@ -496,17 +496,20 @@ func FetchCommonDeps() {
 var NewDepRepoVFS = func(ctx context.Context, cloneURL *url.URL, rev string, zipURLTemplate *string) (ctxvfs.FileSystem, error) {
 	if zipURLTemplate != nil {
 		zipURL := fmt.Sprintf(*zipURLTemplate, path.Join(cloneURL.Host, cloneURL.Path), rev)
-		response, err := http.Head(zipURL)
-		if err == nil && response.StatusCode == http.StatusOK {
-			return vfsutil.NewZipVFS(zipURL, depZipFetch.Inc, depZipFetchFailed.Inc, false)
+		archive, err := vfsutil.NewZipVFS(ctx, zipURL, depZipFetch.Inc, depZipFetchFailed.Inc, false)
+		if archive != nil && err == nil {
+			return archive, nil
 		}
 	}
 
 	// Fast-path for GitHub repos, which we can fetch on-demand from
 	// GitHub's repo .zip archive download endpoint.
 	if cloneURL.Host == "github.com" {
 		fullName := cloneURL.Host + strings.TrimSuffix(cloneURL.Path, ".git") // of the form "github.com/foo/bar"
-		return vfsutil.NewGitHubRepoVFS(fullName, rev)
+		archive, err := vfsutil.NewGitHubRepoVFS(ctx, fullName, rev)
+		if archive != nil && err == nil {
+			return archive, nil
+		}
 	}
 
 	// Fall back to a full git clone for non-github.com repos.

buildserver/deps.go

@@ -275,7 +275,7 @@ func useGithubForVFS() func() {
 		if u.Rev() == "" {
 			return nil, errors.Errorf("rev is required in uri: %s", initializeParams.OriginalRootURI)
 		}
-		return vfsutil.NewGitHubRepoVFS(string(u.Repo()), u.Rev())
+		return vfsutil.NewGitHubRepoVFS(ctx, string(u.Repo()), u.Rev())
 	}
 
 	return func() {

buildserver/integration_test.go

@@ -27,7 +27,7 @@ var RemoteFS = func(ctx context.Context, initializeParams lspext.InitializeParam
 		return url
 	}()
 	if zipURL != "" {
-		return vfsutil.NewZipVFS(zipURL, zipFetch.Inc, zipFetchFailed.Inc, true)
+		return vfsutil.NewZipVFS(ctx, zipURL, zipFetch.Inc, zipFetchFailed.Inc, true)
 	}
 	return nil, errors.Errorf("no zipURL was provided in the initializeOptions")
 }

buildserver/vfs.go

@@ -1,6 +1,7 @@
 package vfsutil
 
 import (
+	"context"
 	"fmt"
 	"regexp"
 
@@ -9,13 +10,13 @@ import (
 
 // NewGitHubRepoVFS creates a new VFS backed by a GitHub downloadable
 // repository archive.
-func NewGitHubRepoVFS(repo, rev string) (*ArchiveFS, error) {
+func NewGitHubRepoVFS(ctx context.Context, repo, rev string) (*ArchiveFS, error) {
 	if !githubRepoRx.MatchString(repo) {
 		return nil, fmt.Errorf(`invalid GitHub repo %q: must be "github.com/user/repo"`, repo)
 	}
 
 	url := fmt.Sprintf("https://codeload.%s/zip/%s", repo, rev)
-	return NewZipVFS(url, ghFetch.Inc, ghFetchFailed.Inc, false)
+	return NewZipVFS(ctx, url, ghFetch.Inc, ghFetchFailed.Inc, false)
 }
 
 var githubRepoRx = regexp.MustCompile(`^github\.com/[\w.-]{1,100}/[\w.-]{1,100}$`)

vfsutil/github_archive.go

@@ -1,6 +1,7 @@
 package vfsutil
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -12,7 +13,7 @@ func TestGitHubRepoVFS(t *testing.T) {
 	defer cleanup()
 
 	// Any public repo will work.
-	fs, err := NewGitHubRepoVFS("github.com/gorilla/schema", "0164a00ab4cd01d814d8cd5bf63fd9fcea30e23b")
+	fs, err := NewGitHubRepoVFS(context.Background(), "github.com/gorilla/schema", "0164a00ab4cd01d814d8cd5bf63fd9fcea30e23b")
 	if err != nil {
 		t.Fatal(err)
 	}

vfsutil/github_archive_test.go

@@ -2,11 +2,16 @@ package vfsutil
 
 import (
 	"context"
+	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"strings"
 
 	"golang.org/x/net/context/ctxhttp"
 
+	"github.com/fhs/go-netrc/netrc"
 	"github.com/pkg/errors"
 
 	opentracing "github.com/opentracing/opentracing-go"
@@ -15,7 +20,20 @@ import (
 
 // NewZipVFS downloads a zip archive from a URL (or fetches from the local cache
 // on disk) and returns a new VFS backed by that zip archive.
-func NewZipVFS(url string, onFetchStart, onFetchFailed func(), evictOnClose bool) (*ArchiveFS, error) {
+func NewZipVFS(ctx context.Context, url string, onFetchStart, onFetchFailed func(), evictOnClose bool) (*ArchiveFS, error) {
+	request, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to construct a new request with URL %s", url)
+	}
+	setAuthFromNetrc(request)
+	response, err := ctxhttp.Do(ctx, nil, request)
+	if err != nil {
+		return nil, err
+	}
+	if response.StatusCode != http.StatusOK {
+		return nil, nil
+	}
+
 	fetch := func(ctx context.Context) (ar *archiveReader, err error) {
 		span, ctx := opentracing.StartSpanFromContext(ctx, "zip Fetch")
 		ext.Component.Set(span, "zipvfs")
@@ -35,6 +53,7 @@ func NewZipVFS(url string, onFetchStart, onFetchFailed func(), evictOnClose bool
 				return nil, errors.Wrapf(err, "failed to construct a new request with URL %s", url)
 			}
 			request.Header.Add("Accept", "application/zip")
+			setAuthFromNetrc(request)
 			resp, err := ctxhttp.Do(ctx, nil, request)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to fetch zip archive from %s", url)
@@ -71,3 +90,19 @@ func NewZipVFS(url string, onFetchStart, onFetchFailed func(), evictOnClose bool
 
 	return &ArchiveFS{fetch: fetch, EvictOnClose: evictOnClose}, nil
 }
+
+func setAuthFromNetrc(req *http.Request) {
+	host := req.URL.Host
+	if i := strings.Index(host, ":"); i != -1 {
+		host = host[:i]
+	}
+	netrcFile := os.ExpandEnv("$HOME/.netrc")
+	if _, err := os.Stat(netrcFile); os.IsNotExist(err) {
+		return
+	}
+	machine, err := netrc.FindMachine(netrcFile, host)
+	if err != nil || machine == nil {
+		return
+	}
+	req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", machine.Login, machine.Password))))
+}