Merge branch '217392-read-references' into 'master'

Read and parse LSIF references

See merge request gitlab-org/gitlab-workhorse!524

internal/api/api.go

@@ -145,6 +145,8 @@ type Response struct {
 	ShowAllRefs bool
 	// Detects whether an artifact is used for code intelligence
 	ProcessLsif bool
+	// Detects whether LSIF artifact will be parsed with references
+	ProcessLsifReferences bool
 }
 
 // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy

internal/lsif_transformer/parser/docs.go

@@ -31,8 +31,8 @@ type Metadata struct {
 	Root string `json:"projectRoot"`
 }
 
-func NewDocs(tempDir string) (*Docs, error) {
-	ranges, err := NewRanges(tempDir)
+func NewDocs(config Config) (*Docs, error) {
+	ranges, err := NewRanges(config)
 	if err != nil {
 		return nil, err
 	}

internal/lsif_transformer/parser/docs_test.go

@@ -12,7 +12,7 @@ func createLine(id, label, uri string) []byte {
 }
 
 func TestRead(t *testing.T) {
-	d, err := NewDocs("")
+	d, err := NewDocs(Config{ProcessReferences: true})
 	require.NoError(t, err)
 	defer d.Close()
 
@@ -29,7 +29,7 @@ func TestRead(t *testing.T) {
 }
 
 func TestReadContainsLine(t *testing.T) {
-	d, err := NewDocs("")
+	d, err := NewDocs(Config{ProcessReferences: true})
 	require.NoError(t, err)
 	defer d.Close()
 

internal/lsif_transformer/parser/hovers.go

@@ -36,8 +36,10 @@ type ResultSetRef struct {
 	RefId       Id `json:"inV"`
 }
 
-func NewHovers(tempDir string) (*Hovers, error) {
-	file, err := ioutil.TempFile(tempDir, "hovers")
+func NewHovers(config Config) (*Hovers, error) {
+	tempPath := config.TempPath
+
+	file, err := ioutil.TempFile(tempPath, "hovers")
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +48,7 @@ func NewHovers(tempDir string) (*Hovers, error) {
 		return nil, err
 	}
 
-	offsets, err := newCache(tempDir, "hovers-indexes", Offset{})
+	offsets, err := newCache(tempPath, "hovers-indexes", Offset{})
 	if err != nil {
 		return nil, err
 	}

internal/lsif_transformer/parser/hovers_test.go

@@ -19,7 +19,7 @@ func TestHoversRead(t *testing.T) {
 }
 
 func setupHovers(t *testing.T) *Hovers {
-	h, err := NewHovers("")
+	h, err := NewHovers(Config{})
 	require.NoError(t, err)
 
 	require.NoError(t, h.Read("hoverResult", []byte(`{"id":"2","label":"hoverResult","result":{"contents": ["hello"]}}`)))

internal/lsif_transformer/parser/parser.go

@@ -18,13 +18,18 @@ type Parser struct {
 	Docs *Docs
 }
 
-func NewParser(r io.Reader, tempDir string) (*Parser, error) {
-	docs, err := NewDocs(tempDir)
+type Config struct {
+	TempPath          string
+	ProcessReferences bool
+}
+
+func NewParser(r io.Reader, config Config) (*Parser, error) {
+	docs, err := NewDocs(config)
 	if err != nil {
 		return nil, err
 	}
 
-	zr, err := openZipReader(r, tempDir)
+	zr, err := openZipReader(r, config.TempPath)
 	if err != nil {
 		return nil, err
 	}

internal/lsif_transformer/parser/parser_test.go

@@ -41,7 +41,7 @@ func createFiles(t *testing.T, filePath, tmpDir string) {
 	file, err := os.Open(filePath)
 	require.NoError(t, err)
 
-	p, err := NewParser(file, "")
+	p, err := NewParser(file, Config{ProcessReferences: true})
 	require.NoError(t, err)
 
 	r, err := p.ZipReader()

internal/lsif_transformer/parser/performance_test.go

@@ -18,7 +18,7 @@ func BenchmarkGenerate(b *testing.B) {
 		file, err := os.Open(filePath)
 		require.NoError(b, err)
 
-		p, err := NewParser(file, "")
+		p, err := NewParser(file, Config{ProcessReferences: true})
 		require.NoError(b, err)
 
 		_, err = p.ZipReader()

internal/lsif_transformer/parser/ranges.go

@@ -14,8 +14,10 @@ const (
 
 type Ranges struct {
 	DefRefs           map[Id]Item
+	References        map[Id][]Item
 	Hovers            *Hovers
 	Cache             *cache
+	ProcessReferences bool
 }
 
 type RawRange struct {
@@ -46,23 +48,30 @@ type SerializedRange struct {
 	StartChar      int32                 `json:"start_char"`
 	DefinitionPath string                `json:"definition_path,omitempty"`
 	Hover          json.RawMessage       `json:"hover"`
+	References     []SerializedReference `json:"references,omitempty"`
 }
 
-func NewRanges(tempDir string) (*Ranges, error) {
-	hovers, err := NewHovers(tempDir)
+type SerializedReference struct {
+	Path string `json:"path"`
+}
+
+func NewRanges(config Config) (*Ranges, error) {
+	hovers, err := NewHovers(config)
 	if err != nil {
 		return nil, err
 	}
 
-	cache, err := newCache(tempDir, "ranges", Range{})
+	cache, err := newCache(config.TempPath, "ranges", Range{})
 	if err != nil {
 		return nil, err
 	}
 
 	return &Ranges{
 		DefRefs:           make(map[Id]Item),
+		References:        make(map[Id][]Item),
 		Hovers:            hovers,
 		Cache:             cache,
+		ProcessReferences: config.ProcessReferences,
 	}, nil
 }
 
@@ -102,6 +111,7 @@ func (r *Ranges) Serialize(f io.Writer, rangeIds []Id, docs map[Id]string) error
 			StartChar:      entry.Character,
 			DefinitionPath: r.definitionPathFor(docs, entry.RefId),
 			Hover:          r.Hovers.For(entry.RefId),
+			References:     r.referencesFor(docs, entry.RefId),
 		}
 		if err := encoder.Encode(serializedRange); err != nil {
 			return err
@@ -138,6 +148,29 @@ func (r *Ranges) definitionPathFor(docs map[Id]string, refId Id) string {
 	return defPath
 }
 
+func (r *Ranges) referencesFor(docs map[Id]string, refId Id) []SerializedReference {
+	if !r.ProcessReferences {
+		return nil
+	}
+
+	references, ok := r.References[refId]
+	if !ok {
+		return nil
+	}
+
+	var serializedReferences []SerializedReference
+
+	for _, reference := range references {
+		serializedReference := SerializedReference{
+			Path: docs[reference.DocId] + "#L" + reference.Line,
+		}
+
+		serializedReferences = append(serializedReferences, serializedReference)
+	}
+
+	return serializedReferences
+}
+
 func (r *Ranges) addRange(line []byte) error {
 	var rg RawRange
 	if err := json.Unmarshal(line, &rg); err != nil {
@@ -157,6 +190,10 @@ func (r *Ranges) addItem(line []byte) error {
 		return nil
 	}
 
+	if len(rawItem.RangeIds) == 0 {
+		return errors.New("no range IDs")
+	}
+
 	for _, rangeId := range rawItem.RangeIds {
 		rg, err := r.getRange(rangeId)
 		if err != nil {
@@ -168,28 +205,17 @@ func (r *Ranges) addItem(line []byte) error {
 		if err := r.Cache.SetEntry(rangeId, rg); err != nil {
 			return err
 		}
-	}
-
-	if rawItem.Property == definitions {
-		return r.addDefRef(&rawItem)
-	}
 
-	return nil
-}
-
-func (r *Ranges) addDefRef(rawItem *RawItem) error {
-	if len(rawItem.RangeIds) == 0 {
-		return errors.New("no range IDs")
+		item := Item{
+			Line:  strconv.Itoa(int(rg.Line + 1)),
+			DocId: rawItem.DocId,
 		}
 
-	rg, err := r.getRange(rawItem.RangeIds[0])
-	if err != nil {
-		return err
+		if rawItem.Property == definitions {
+			r.DefRefs[rawItem.RefId] = item
+		} else if r.ProcessReferences {
+			r.References[rawItem.RefId] = append(r.References[rawItem.RefId], item)
 		}
-
-	r.DefRefs[rawItem.RefId] = Item{
-		Line:  strconv.Itoa(int(rg.Line + 1)),
-		DocId: rawItem.DocId,
 	}
 
 	return nil

internal/lsif_transformer/parser/ranges_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestRangesRead(t *testing.T) {
-	r, cleanup := setup(t)
+	r, cleanup := setup(t, true)
 	defer cleanup()
 
 	firstRange := Range{Line: 1, Character: 2, RefId: 3}
@@ -23,10 +23,24 @@ func TestRangesRead(t *testing.T) {
 }
 
 func TestSerialize(t *testing.T) {
-	r, cleanup := setup(t)
+	r, cleanup := setup(t, true)
 	defer cleanup()
 
-	docs := map[Id]string{6: "def-path"}
+	docs := map[Id]string{6: "def-path", 7: "ref-path"}
+
+	var buf bytes.Buffer
+	err := r.Serialize(&buf, []Id{1}, docs)
+	want := `[{"start_line":1,"start_char":2,"definition_path":"def-path#L2","hover":null,"references":[{"path":"ref-path#L6"}]}` + "\n]"
+
+	require.NoError(t, err)
+	require.Equal(t, want, buf.String())
+}
+
+func TestSerializeWithoutProcessingReferences(t *testing.T) {
+	r, cleanup := setup(t, false)
+	defer cleanup()
+
+	docs := map[Id]string{6: "def-path", 7: "ref-path"}
 
 	var buf bytes.Buffer
 	err := r.Serialize(&buf, []Id{1}, docs)
@@ -36,15 +50,15 @@ func TestSerialize(t *testing.T) {
 	require.Equal(t, want, buf.String())
 }
 
-func setup(t *testing.T) (*Ranges, func()) {
-	r, err := NewRanges("")
+func setup(t *testing.T, processReferences bool) (*Ranges, func()) {
+	r, err := NewRanges(Config{ProcessReferences: processReferences})
 	require.NoError(t, err)
 
 	require.NoError(t, r.Read("range", []byte(`{"id":1,"label":"range","start":{"line":1,"character":2}}`)))
 	require.NoError(t, r.Read("range", []byte(`{"id":"2","label":"range","start":{"line":5,"character":4}}`)))
 
 	require.NoError(t, r.Read("item", []byte(`{"id":4,"label":"item","property":"definitions","outV":"3","inVs":[1],"document":"6"}`)))
-	require.NoError(t, r.Read("item", []byte(`{"id":"5","label":"item","property":"references","outV":3,"inVs":["2"]}`)))
+	require.NoError(t, r.Read("item", []byte(`{"id":"5","label":"item","property":"references","outV":3,"inVs":["2"],"document":"7"}`)))
 
 	cleanup := func() {
 		require.NoError(t, r.Close())

internal/lsif_transformer/parser/testdata/expected/lsif/main.go.json

@@ -25,6 +25,14 @@
             {
                 "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L8"
+            },
+            {
+                "path": "main.go#L9"
+            }
         ]
     },
     {
@@ -60,6 +68,11 @@
             {
                 "value": "This method reverses a string \n\n"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L8"
+            }
         ]
     },
     {
@@ -88,6 +101,14 @@
             {
                 "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L8"
+            },
+            {
+                "path": "main.go#L9"
+            }
         ]
     },
     {
@@ -120,6 +141,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L9"
+            }
         ]
     },
     {
@@ -169,6 +195,14 @@
             {
                 "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L8"
+            },
+            {
+                "path": "main.go#L9"
+            }
         ]
     }
 ]
\ No newline at end of file

internal/lsif_transformer/parser/testdata/expected/lsif/morestrings/reverse.go.json

@@ -32,6 +32,11 @@
             {
                 "value": "This method reverses a string \n\n"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L8"
+            }
         ]
     },
     {
@@ -107,6 +112,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "morestrings/reverse.go#L15"
+            }
         ]
     },
     {
@@ -132,6 +142,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "morestrings/reverse.go#L8"
+            }
         ]
     },
     {
@@ -157,6 +172,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "morestrings/reverse.go#L15"
+            }
         ]
     },
     {
@@ -182,6 +202,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "morestrings/reverse.go#L8"
+            }
         ]
     },
     {
@@ -214,6 +239,11 @@
                 ],
                 "language": "go"
             }
+        ],
+        "references": [
+            {
+                "path": "main.go#L9"
+            }
         ]
     }
 ]
\ No newline at end of file

internal/upload/rewrite.go

@@ -133,7 +133,7 @@ func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipa
 			return err
 		}
 	case rew.preauth.ProcessLsif:
-		inputReader, err = handleLsifUpload(ctx, p, opts.LocalTempPath, filename)
+		inputReader, err = handleLsifUpload(ctx, p, opts.LocalTempPath, filename, rew.preauth)
 		if err != nil {
 			return err
 		}
@@ -174,8 +174,13 @@ func handleExifUpload(ctx context.Context, r io.Reader, filename string) (io.Rea
 	return exif.NewCleaner(ctx, r)
 }
 
-func handleLsifUpload(ctx context.Context, reader io.Reader, tempPath, filename string) (io.Reader, error) {
-	p, err := parser.NewParser(reader, tempPath)
+func handleLsifUpload(ctx context.Context, reader io.Reader, tempPath, filename string, preauth *api.Response) (io.Reader, error) {
+	parserConfig := parser.Config{
+		TempPath:          tempPath,
+		ProcessReferences: preauth.ProcessLsifReferences,
+	}
+
+	p, err := parser.NewParser(reader, parserConfig)
 	if err != nil {
 		return nil, err
 	}