gopls/internal/golang: customize semantic token types and modifiers

We agreed to return full set of token types and modifiers from gopls
by default (full means token types and modifiers that gopls understand)
and provide configuraton options for users to disable some of them.

- Two fields of type map[string]bool are introduced to gopls UIOptions
(workspace/configuration) to customize semantic token types and
modifiers. For now, only value of "false" is effective. Choose type of
map over array to keep future compatibility in case we want to introduce
enable capabilities.
- VSCode-Go populate these options from user settings to gopls.
- Gopls "initialize" protocol returns a pre-defined fixed legend
including subset of standard legend defined LSP that gopls understand
with additional customize modifiers gopls recoganized.
- Gopls "textDocument/semanticTokens" protocol returns token types and
modifiers based on configuration defined in workspace/configuration.

Tested with vscode-go changes CL 642416, screenshot is at
golang/vscode-go#3632 (comment)

For golang/vscode-go#3632

Change-Id: Ie8220e12a4c8d6c84c54992d84277767e61ec023
Reviewed-on: https://go-review.googlesource.com/c/tools/+/642077
Auto-Submit: Hongxiang Jiang <hxjiang@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Robert Findley <rfindley@google.com>

Author: h9jiang

gopls/doc/settings.md

@@ -215,10 +215,32 @@ Default: `false`.
 
 **This setting is experimental and may be deleted.**
 
-noSemanticNumber  turns off the sending of the semantic token 'number'
+noSemanticNumber turns off the sending of the semantic token 'number'
 
 Default: `false`.
 
+<a id='semanticTokenTypes'></a>
+### `semanticTokenTypes map[string]bool`
+
+**This setting is experimental and may be deleted.**
+
+semanticTokenTypes configures the semantic token types. It allows
+disabling types by setting each value to false.
+By default, all types are enabled.
+
+Default: `{}`.
+
+<a id='semanticTokenModifiers'></a>
+### `semanticTokenModifiers map[string]bool`
+
+**This setting is experimental and may be deleted.**
+
+semanticTokenModifiers configures the semantic token modifiers. It allows
+disabling modifiers by setting each value to false.
+By default, all modifiers are enabled.
+
+Default: `{}`.
+
 <a id='completion'></a>
 ## Completion
 

gopls/internal/cmd/cmd.go

@@ -27,10 +27,12 @@ import (
 	"golang.org/x/tools/gopls/internal/lsprpc"
 	"golang.org/x/tools/gopls/internal/protocol"
 	"golang.org/x/tools/gopls/internal/protocol/command"
+	"golang.org/x/tools/gopls/internal/protocol/semtok"
 	"golang.org/x/tools/gopls/internal/server"
 	"golang.org/x/tools/gopls/internal/settings"
 	"golang.org/x/tools/gopls/internal/util/browser"
 	bugpkg "golang.org/x/tools/gopls/internal/util/bug"
+	"golang.org/x/tools/gopls/internal/util/moreslices"
 	"golang.org/x/tools/internal/diff"
 	"golang.org/x/tools/internal/jsonrpc2"
 	"golang.org/x/tools/internal/tool"
@@ -299,7 +301,7 @@ func (app *Application) featureCommands() []tool.Application {
 		&prepareRename{app: app},
 		&references{app: app},
 		&rename{app: app},
-		&semtok{app: app},
+		&semanticToken{app: app},
 		&signature{app: app},
 		&stats{app: app},
 		&symbols{app: app},
@@ -322,7 +324,6 @@ func (app *Application) connect(ctx context.Context) (*connection, error) {
 		options := settings.DefaultOptions(app.options)
 		svr = server.New(cache.NewSession(ctx, cache.New(nil)), client, options)
 		ctx = protocol.WithClient(ctx, client)
-
 	} else {
 		// remote
 		netConn, err := lsprpc.ConnectToRemote(ctx, app.Remote)
@@ -362,8 +363,8 @@ func (c *connection) initialize(ctx context.Context, options func(*settings.Opti
 	params.Capabilities.TextDocument.SemanticTokens.Requests.Range = &protocol.Or_ClientSemanticTokensRequestOptions_range{Value: true}
 	//params.Capabilities.TextDocument.SemanticTokens.Requests.Range.Value = true
 	params.Capabilities.TextDocument.SemanticTokens.Requests.Full = &protocol.Or_ClientSemanticTokensRequestOptions_full{Value: true}
-	params.Capabilities.TextDocument.SemanticTokens.TokenTypes = protocol.SemanticTypes()
-	params.Capabilities.TextDocument.SemanticTokens.TokenModifiers = protocol.SemanticModifiers()
+	params.Capabilities.TextDocument.SemanticTokens.TokenTypes = moreslices.ConvertStrings[string](semtok.TokenTypes)
+	params.Capabilities.TextDocument.SemanticTokens.TokenModifiers = moreslices.ConvertStrings[string](semtok.TokenModifiers)
 	params.Capabilities.TextDocument.CodeAction = protocol.CodeActionClientCapabilities{
 		CodeActionLiteralSupport: protocol.ClientCodeActionLiteralOptions{
 			CodeActionKind: protocol.ClientCodeActionKindOptions{
@@ -376,7 +377,7 @@ func (c *connection) initialize(ctx context.Context, options func(*settings.Opti
 	params.InitializationOptions = map[string]interface{}{
 		"symbolMatcher": string(opts.SymbolMatcher),
 	}
-	if _, err := c.Server.Initialize(ctx, params); err != nil {
+	if c.initializeResult, err = c.Server.Initialize(ctx, params); err != nil {
 		return err
 	}
 	if err := c.Server.Initialized(ctx, &protocol.InitializedParams{}); err != nil {
@@ -388,6 +389,9 @@ func (c *connection) initialize(ctx context.Context, options func(*settings.Opti
 type connection struct {
 	protocol.Server
 	client *cmdClient
+	// initializeResult keep the initialize protocol response from server
+	// including server capabilities.
+	initializeResult *protocol.InitializeResult
 }
 
 // cmdClient defines the protocol.Client interface behavior of the gopls CLI tool.

gopls/internal/cmd/semantictokens.go

@@ -14,6 +14,7 @@ import (
 	"unicode/utf8"
 
 	"golang.org/x/tools/gopls/internal/protocol"
+	"golang.org/x/tools/gopls/internal/protocol/semtok"
 	"golang.org/x/tools/gopls/internal/settings"
 )
 
@@ -40,15 +41,15 @@ import (
 //    0-based: lines and character positions are 1 less than in
 //      the gopls coordinate system
 
-type semtok struct {
+type semanticToken struct {
 	app *Application
 }
 
-func (c *semtok) Name() string      { return "semtok" }
-func (c *semtok) Parent() string    { return c.app.Name() }
-func (c *semtok) Usage() string     { return "<filename>" }
-func (c *semtok) ShortHelp() string { return "show semantic tokens for the specified file" }
-func (c *semtok) DetailedHelp(f *flag.FlagSet) {
+func (c *semanticToken) Name() string      { return "semtok" }
+func (c *semanticToken) Parent() string    { return c.app.Name() }
+func (c *semanticToken) Usage() string     { return "<filename>" }
+func (c *semanticToken) ShortHelp() string { return "show semantic tokens for the specified file" }
+func (c *semanticToken) DetailedHelp(f *flag.FlagSet) {
 	fmt.Fprint(f.Output(), `
 Example: show the semantic tokens for this file:
 
@@ -59,7 +60,7 @@ Example: show the semantic tokens for this file:
 
 // Run performs the semtok on the files specified by args and prints the
 // results to stdout in the format described above.
-func (c *semtok) Run(ctx context.Context, args ...string) error {
+func (c *semanticToken) Run(ctx context.Context, args ...string) error {
 	if len(args) != 1 {
 		return fmt.Errorf("expected one file name, got %d", len(args))
 	}
@@ -97,14 +98,16 @@ func (c *semtok) Run(ctx context.Context, args ...string) error {
 	if err != nil {
 		return err
 	}
-	return decorate(file, resp.Data)
+	return decorate(conn.initializeResult.Capabilities.SemanticTokensProvider.(protocol.SemanticTokensOptions).Legend, file, resp.Data)
 }
 
+// mark provides a human-readable representation of protocol.SemanticTokens.
+// It translates token types and modifiers to strings instead of uint32 values.
 type mark struct {
 	line, offset int // 1-based, from RangeSpan
 	len          int // bytes, not runes
-	typ          string
-	mods         []string
+	typ          semtok.Type
+	mods         []semtok.Modifier
 }
 
 // prefixes for semantic token comments
@@ -136,8 +139,10 @@ func markLine(m mark, lines [][]byte) {
 	lines[m.line-1] = l
 }
 
-func decorate(file *cmdFile, result []uint32) error {
-	marks := newMarks(file, result)
+// decorate translates semantic token data (protocol.SemanticTokens) from its
+// raw []uint32 format into a human-readable representation and prints it to stdout.
+func decorate(legend protocol.SemanticTokensLegend, file *cmdFile, data []uint32) error {
+	marks := newMarks(legend, file, data)
 	if len(marks) == 0 {
 		return nil
 	}
@@ -150,45 +155,56 @@ func decorate(file *cmdFile, result []uint32) error {
 	return nil
 }
 
-func newMarks(file *cmdFile, d []uint32) []mark {
+func newMarks(legend protocol.SemanticTokensLegend, file *cmdFile, data []uint32) []mark {
 	ans := []mark{}
 	// the following two loops could be merged, at the cost
 	// of making the logic slightly more complicated to understand
 	// first, convert from deltas to absolute, in LSP coordinates
-	lspLine := make([]uint32, len(d)/5)
-	lspChar := make([]uint32, len(d)/5)
+	lspLine := make([]uint32, len(data)/5)
+	lspChar := make([]uint32, len(data)/5)
 	var line, char uint32
-	for i := 0; 5*i < len(d); i++ {
-		lspLine[i] = line + d[5*i+0]
-		if d[5*i+0] > 0 {
+	for i := 0; 5*i < len(data); i++ {
+		lspLine[i] = line + data[5*i+0]
+		if data[5*i+0] > 0 {
 			char = 0
 		}
-		lspChar[i] = char + d[5*i+1]
+		lspChar[i] = char + data[5*i+1]
 		char = lspChar[i]
 		line = lspLine[i]
 	}
 	// second, convert to gopls coordinates
-	for i := 0; 5*i < len(d); i++ {
+	for i := 0; 5*i < len(data); i++ {
 		pr := protocol.Range{
 			Start: protocol.Position{
 				Line:      lspLine[i],
 				Character: lspChar[i],
 			},
 			End: protocol.Position{
 				Line:      lspLine[i],
-				Character: lspChar[i] + d[5*i+2],
+				Character: lspChar[i] + data[5*i+2],
 			},
 		}
 		spn, err := file.rangeSpan(pr)
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		var mods []semtok.Modifier
+		{
+			n := int(data[5*i+4])
+			for i, mod := range legend.TokenModifiers {
+				if (n & (1 << i)) != 0 {
+					mods = append(mods, semtok.Modifier(mod))
+				}
+			}
+		}
+
 		m := mark{
 			line:   spn.Start().Line(),
 			offset: spn.Start().Column(),
 			len:    spn.End().Column() - spn.Start().Column(),
-			typ:    protocol.SemType(int(d[5*i+3])),
-			mods:   protocol.SemMods(int(d[5*i+4])),
+			typ:    semtok.Type(legend.TokenTypes[data[5*i+3]]),
+			mods:   mods,
 		}
 		ans = append(ans, m)
 	}

gopls/internal/doc/api.json

@@ -847,7 +847,7 @@
 			{
 				"Name": "noSemanticNumber",
 				"Type": "bool",
-				"Doc": "noSemanticNumber  turns off the sending of the semantic token 'number'\n",
+				"Doc": "noSemanticNumber turns off the sending of the semantic token 'number'\n",
 				"EnumKeys": {
 					"ValueType": "",
 					"Keys": null
@@ -857,6 +857,32 @@
 				"Status": "experimental",
 				"Hierarchy": "ui"
 			},
+			{
+				"Name": "semanticTokenTypes",
+				"Type": "map[string]bool",
+				"Doc": "semanticTokenTypes configures the semantic token types. It allows\ndisabling types by setting each value to false.\nBy default, all types are enabled.\n",
+				"EnumKeys": {
+					"ValueType": "",
+					"Keys": null
+				},
+				"EnumValues": null,
+				"Default": "{}",
+				"Status": "experimental",
+				"Hierarchy": "ui"
+			},
+			{
+				"Name": "semanticTokenModifiers",
+				"Type": "map[string]bool",
+				"Doc": "semanticTokenModifiers configures the semantic token modifiers. It allows\ndisabling modifiers by setting each value to false.\nBy default, all modifiers are enabled.\n",
+				"EnumKeys": {
+					"ValueType": "",
+					"Keys": null
+				},
+				"EnumValues": null,
+				"Default": "{}",
+				"Status": "experimental",
+				"Hierarchy": "ui"
+			},
 			{
 				"Name": "local",
 				"Type": "string",

gopls/internal/golang/semtok.go

@@ -82,10 +82,8 @@ func SemanticTokens(ctx context.Context, snapshot *cache.Snapshot, fh file.Handl
 	return &protocol.SemanticTokens{
 		Data: semtok.Encode(
 			tv.tokens,
-			snapshot.Options().NoSemanticString,
-			snapshot.Options().NoSemanticNumber,
-			snapshot.Options().SemanticTypes,
-			snapshot.Options().SemanticMods),
+			snapshot.Options().EnabledSemanticTokenTypes(),
+			snapshot.Options().EnabledSemanticTokenModifiers()),
 		ResultID: time.Now().String(), // for delta requests, but we've never seen any
 	}, nil
 }
@@ -242,7 +240,7 @@ func (tv *tokenVisitor) comment(c *ast.Comment, importByName map[string]*types.P
 }
 
 // token emits a token of the specified extent and semantics.
-func (tv *tokenVisitor) token(start token.Pos, length int, typ semtok.TokenType, modifiers ...semtok.Modifier) {
+func (tv *tokenVisitor) token(start token.Pos, length int, typ semtok.Type, modifiers ...semtok.Modifier) {
 	if !start.IsValid() {
 		return
 	}
@@ -463,7 +461,7 @@ func (tv *tokenVisitor) inspect(n ast.Node) (descend bool) {
 	return true
 }
 
-func (tv *tokenVisitor) appendObjectModifiers(mods []semtok.Modifier, obj types.Object) (semtok.TokenType, []semtok.Modifier) {
+func (tv *tokenVisitor) appendObjectModifiers(mods []semtok.Modifier, obj types.Object) (semtok.Type, []semtok.Modifier) {
 	if obj.Pkg() == nil {
 		mods = append(mods, semtok.ModDefaultLibrary)
 	}
@@ -559,7 +557,7 @@ func appendTypeModifiers(mods []semtok.Modifier, t types.Type) []semtok.Modifier
 
 func (tv *tokenVisitor) ident(id *ast.Ident) {
 	var (
-		tok  semtok.TokenType
+		tok  semtok.Type
 		mods []semtok.Modifier
 		obj  types.Object
 		ok   bool
@@ -623,7 +621,7 @@ func (tv *tokenVisitor) isParam(pos token.Pos) bool {
 // def), use the parse stack.
 // A lot of these only happen when the package doesn't compile,
 // but in that case it is all best-effort from the parse tree.
-func (tv *tokenVisitor) unkIdent(id *ast.Ident) (semtok.TokenType, []semtok.Modifier) {
+func (tv *tokenVisitor) unkIdent(id *ast.Ident) (semtok.Type, []semtok.Modifier) {
 	def := []semtok.Modifier{semtok.ModDefinition}
 	n := len(tv.stack) - 2 // parent of Ident; stack is [File ... Ident]
 	if n < 0 {
@@ -746,7 +744,7 @@ func (tv *tokenVisitor) unkIdent(id *ast.Ident) (semtok.TokenType, []semtok.Modi
 }
 
 // multiline emits a multiline token (`string` or /*comment*/).
-func (tv *tokenVisitor) multiline(start, end token.Pos, tok semtok.TokenType) {
+func (tv *tokenVisitor) multiline(start, end token.Pos, tok semtok.Type) {
 	// TODO(adonovan): test with non-ASCII.
 
 	f := tv.fset.File(start)