Skip to content

G
gitlab-ce
Commit 397c4c36 authored by Nick Thomas's avatar Nick Thomas
Browse files
Options

Merge branch 'id-send-tokens-instead-of-html' into 'master' 

Send hover tokens instead of raw html

See merge request gitlab-org/gitlab-workhorse!517
parents 4bb78061 c02e372a
Hide whitespace changes
Inline Side-by-side
Showing with 309 additions and 102 deletions
internal/lsif_transformer/parser/code_hover.go
View file @ 397c4c36
package parser
import (
"bytes"
"encoding/json"
"html/template"
"io"
"strings"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/lexers"
)
var (
languageTemplate = template.Must(template.New("lang").Parse(`<span class="line" lang="{{.}}">`))
valueTemplate = template.Must(template.New("value").Parse(`<span class="{{.Class}}">{{.Value}}</span>`))
)
type token struct {
Class string `json:"class,omitempty"`
Value string `json:"value"`
}
type CodeHover struct {
Value string `json:"value"`
Language string `json:"language,omitempty"`
type codeHover struct {
Value string `json:"value,omitempty"`
Tokens [][]token `json:"tokens,omitempty"`
Language string `json:"language,omitempty"`
}
func NewCodeHover(content json.RawMessage) (*CodeHover, error) {
func newCodeHover(content json.RawMessage) (*codeHover, error) {
// Hover value can be either an object: { "value": "func main()", "language": "go" }
// Or a string with documentation
// We try to unmarshal the content into a string and if we fail, we unmarshal it into an object
var codeHover CodeHover
if err := json.Unmarshal(content, &codeHover.Value); err != nil {
if err := json.Unmarshal(content, &codeHover); err != nil {
var c codeHover
if err := json.Unmarshal(content, &c.Value); err != nil {
if err := json.Unmarshal(content, &c); err != nil {
return nil, err
}
codeHover.Highlight()
c.setTokens()
}
return &codeHover, nil
return &c, nil
}
func (c *CodeHover) Highlight() {
var b bytes.Buffer
for i, line := range c.codeLines() {
if i > 0 {
if _, err := io.WriteString(&b, "\n"); err != nil {
return
}
}
func (c *codeHover) setTokens() {
lexer := lexers.Get(c.Language)
if lexer == nil {
return
}
languageTemplate.Execute(&b, c.Language)
iterator, err := lexer.Tokenise(nil, c.Value)
if err != nil {
return
}
for _, token := range line {
if err := writeTokenValue(&b, token); err != nil {
return
var tokenLines [][]token
for _, tokenLine := range chroma.SplitTokensIntoLines(iterator.Tokens()) {
var tokens []token
var rawToken string
for _, t := range tokenLine {
class := c.classFor(t.Type)
// accumulate consequent raw values in a single string to store them as
// [{ Class: "kd", Value: "func" }, { Value: " main() {" }] instead of
// [{ Class: "kd", Value: "func" }, { Value: " " }, { Value: "main" }, { Value: "(" }...]
if class == "" {
rawToken = rawToken + t.Value
} else {
if rawToken != "" {
tokens = append(tokens, token{Value: rawToken})
rawToken = ""
}
tokens = append(tokens, token{Class: class, Value: t.Value})
}
}
if _, err := io.WriteString(&b, "</span>"); err != nil {
return
if rawToken != "" {
tokens = append(tokens, token{Value: rawToken})
}
}
c.Value = b.String()
}
func writeTokenValue(w io.Writer, token chroma.Token) error {
if strings.HasPrefix(token.Type.String(), "Keyword") || token.Type == chroma.String || token.Type == chroma.Comment {
data := struct {
Class string
Value string
}{
Class: chroma.StandardTypes[token.Type],
Value: replaceNewLines(token.Value),
}
return valueTemplate.Execute(w, data)
tokenLines = append(tokenLines, tokens)
}
_, err := io.WriteString(w, template.HTMLEscapeString(replaceNewLines(token.Value)))
return err
c.Tokens = tokenLines
c.Value = ""
}
func replaceNewLines(value string) string {
return strings.ReplaceAll(value, "\n", "")
}
func (c *CodeHover) codeLines() [][]chroma.Token {
lexer := lexers.Get(c.Language)
if lexer == nil {
return [][]chroma.Token{}
}
iterator, err := lexer.Tokenise(nil, c.Value)
if err != nil {
return [][]chroma.Token{}
func (c *codeHover) classFor(tokenType chroma.TokenType) string {
if strings.HasPrefix(tokenType.String(), "Keyword") || tokenType == chroma.String || tokenType == chroma.Comment {
return chroma.StandardTypes[tokenType]
}
return chroma.SplitTokensIntoLines(iterator.Tokens())
return ""
}
internal/lsif_transformer/parser/code_hover_test.go
View file @ 397c4c36
......@@ -13,71 +13,68 @@ func TestHighlight(t *testing.T) {
name string
language string
value string
want string
want [][]token
}{
{
name: "go function definition",
language: "go",
value: "func main()",
want: "<span class=\"line\" lang=\"go\"><span class=\"kd\">func</span> main()</span>",
want: [][]token{{{Class: "kd", Value: "func"}, {Value: " main()"}}},
},
{
name: "go struct definition",
language: "go",
value: "type Command struct",
want: "<span class=\"line\" lang=\"go\"><span class=\"kd\">type</span> Command <span class=\"kd\">struct</span></span>",
want: [][]token{{{Class: "kd", Value: "type"}, {Value: " Command "}, {Class: "kd", Value: "struct"}}},
},
{
name: "go struct multiline definition",
language: "go",
value: `struct {\nConfig *Config\nReadWriter *ReadWriter\nEOFSent bool\n}`,
want: "<span class=\"line\" lang=\"go\"><span class=\"kd\">struct</span> {</span>\n<span class=\"line\" lang=\"go\">Config *Config</span>\n<span class=\"line\" lang=\"go\">ReadWriter *ReadWriter</span>\n<span class=\"line\" lang=\"go\">EOFSent <span class=\"kt\">bool</span></span>\n<span class=\"line\" lang=\"go\">}</span>",
want: [][]token{
{{Class: "kd", Value: "struct"}, {Value: " {\n"}},
{{Value: "Config *Config\n"}},
{{Value: "ReadWriter *ReadWriter\n"}},
{{Value: "EOFSent "}, {Class: "kt", Value: "bool"}, {Value: "\n"}},
{{Value: "}"}},
},
},
{
name: "ruby method definition",
language: "ruby",
value: "def read(line)",
want: "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> read(line)</span>",
want: [][]token{{{Class: "k", Value: "def"}, {Value: " read(line)"}}},
},
{
name: "amp symbol is escaped",
name: "ruby multiline method definition",
language: "ruby",
value: `def &(line)\nend`,
want: "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &amp;(line)</span>\n<span class=\"line\" lang=\"ruby\"><span class=\"k\">end</span></span>",
},
{
name: "less symbol is escaped",
language: "ruby",
value: "def <(line)",
want: "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &lt;(line)</span>",
},
{
name: "more symbol is escaped",
language: "ruby",
value: `def >(line)\nend`,
want: "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &gt;(line)</span>\n<span class=\"line\" lang=\"ruby\"><span class=\"k\">end</span></span>",
value: `def read(line)\nend`,
want: [][]token{
{{Class: "k", Value: "def"}, {Value: " read(line)\n"}},
{{Class: "k", Value: "end"}},
},
},
{
name: "unknown/malicious language is passed",
language: "<lang> alert(1); </lang>",
value: `def a;\nend`,
want: "",
want: [][]token(nil),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
raw := []byte(fmt.Sprintf(`{"language":"%s","value":"%s"}`, tt.language, tt.value))
c, err := NewCodeHover(json.RawMessage(raw))
c, err := newCodeHover(json.RawMessage(raw))
require.NoError(t, err)
require.Equal(t, tt.want, c.Value)
require.Equal(t, tt.want, c.Tokens)
})
}
}
func TestMarkdown(t *testing.T) {
value := `"This method reverses a string \n\n"`
c, err := NewCodeHover(json.RawMessage(value))
c, err := newCodeHover(json.RawMessage(value))
require.NoError(t, err)
require.Equal(t, "This method reverses a string \n\n", c.Value)
......
internal/lsif_transformer/parser/hovers.go
View file @ 397c4c36
......@@ -97,14 +97,14 @@ func (h *Hovers) addData(line []byte) error {
return err
}
codeHovers := []*CodeHover{}
codeHovers := []*codeHover{}
for _, rawContent := range rawData.Result.Contents {
codeHover, err := NewCodeHover(rawContent)
c, err := newCodeHover(rawContent)
if err != nil {
return err
}
codeHovers = append(codeHovers, codeHover)
codeHovers = append(codeHovers, c)
}
codeHoversData, err := json.Marshal(codeHovers)
......
internal/lsif_transformer/parser/testdata/expected/lsif/main.go.json
View file @ 397c4c36
......@@ -5,7 +5,21 @@
"definition_path": "main.go#L4",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kn",
"value": "package"
},
{
"value": " "
},
{
"class": "s",
"value": "\"github.com/user/hello/morestrings\""
}
]
],
"language": "go"
},
{
......@@ -19,7 +33,28 @@
"definition_path": "morestrings/reverse.go#L12",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Reverse(s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "func"
},
{
"value": " Reverse(s "
},
{
"class": "kt",
"value": "string"
},
{
"value": ") "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
},
{
......@@ -33,7 +68,21 @@
"definition_path": "main.go#L4",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kn",
"value": "package"
},
{
"value": " "
},
{
"class": "s",
"value": "\"github.com/user/hello/morestrings\""
}
]
],
"language": "go"
},
{
......@@ -47,7 +96,28 @@
"definition_path": "morestrings/reverse.go#L5",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Func2(i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "func"
},
{
"value": " Func2(i "
},
{
"class": "kt",
"value": "int"
},
{
"value": ") "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -58,7 +128,17 @@
"definition_path": "main.go#L7",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e main()\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "func"
},
{
"value": " main()"
}
]
],
"language": "go"
}
]
......@@ -69,7 +149,21 @@
"definition_path": "main.go#L4",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kn",
"value": "package"
},
{
"value": " "
},
{
"class": "s",
"value": "\"github.com/user/hello/morestrings\""
}
]
],
"language": "go"
},
{
......
internal/lsif_transformer/parser/testdata/expected/lsif/morestrings/reverse.go.json
View file @ 397c4c36
......@@ -5,7 +5,28 @@
"definition_path": "morestrings/reverse.go#L12",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Reverse(s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "func"
},
{
"value": " Reverse(s "
},
{
"class": "kt",
"value": "string"
},
{
"value": ") "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
},
{
......@@ -19,7 +40,21 @@
"definition_path": "morestrings/reverse.go#L5",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " i "
},
{
"class": "kt",
"value": "int"
}
]
],
"language": "go"
}
]
......@@ -30,7 +65,21 @@
"definition_path": "morestrings/reverse.go#L12",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " s "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -41,7 +90,21 @@
"definition_path": "morestrings/reverse.go#L13",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e a \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " a "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -52,7 +115,21 @@
"definition_path": "morestrings/reverse.go#L6",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e b \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " b "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -63,7 +140,21 @@
"definition_path": "morestrings/reverse.go#L13",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e a \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " a "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -74,7 +165,21 @@
"definition_path": "morestrings/reverse.go#L6",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e b \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "var"
},
{
"value": " b "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......@@ -85,7 +190,28 @@
"definition_path": "morestrings/reverse.go#L5",
"hover": [
{
"value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Func2(i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
"tokens": [
[
{
"class": "kd",
"value": "func"
},
{
"value": " Func2(i "
},
{
"class": "kt",
"value": "int"
},
{
"value": ") "
},
{
"class": "kt",
"value": "string"
}
]
],
"language": "go"
}
]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7