pkg/manager: export programs with coverage

dashboard/app/public_json_api.go

@@ -12,6 +12,7 @@ import (
 
 	"cloud.google.com/go/civil"
 	"github.com/google/syzkaller/dashboard/api"
+	"github.com/google/syzkaller/pkg/cover"
 	"github.com/google/syzkaller/pkg/coveragedb"
 )
 
@@ -177,26 +178,6 @@ func GetJSONDescrFor(page interface{}) ([]byte, error) {
 	return json.MarshalIndent(res, "", "\t")
 }
 
-type coveredBlock struct {
-	FromLine int `json:"from_line"`
-	FromCol  int `json:"from_column"`
-	ToLine   int `json:"to_line"`
-	ToCol    int `json:"to_column"`
-}
-
-type functionCoverage struct {
-	FuncName     string          `json:"func_name"`
-	Instrumented int             `json:"total_blocks"`
-	Blocks       []*coveredBlock `json:"covered_blocks"`
-}
-
-type fileCoverage struct {
-	Repo      string              `json:"repo"`
-	Commit    string              `json:"commit"`
-	FilePath  string              `json:"file_path"`
-	Functions []*functionCoverage `json:"functions"`
-}
-
 func writeExtAPICoverageFor(ctx context.Context, w io.Writer, ns, repo string) error {
 	// By default, return the previous month coverage. It guarantees the good numbers.
 	//
@@ -236,7 +217,7 @@ func writeFileCoverage(ctx context.Context, w io.Writer, repo string, ff *covera
 			if err != nil {
 				return fmt.Errorf("genFuncsCov: %w", err)
 			}
-			if err := enc.Encode(&fileCoverage{
+			if err := enc.Encode(&cover.FileCoverage{
 				Repo:      repo,
 				Commit:    fileCov.Commit,
 				FilePath:  fileCov.Filepath,
@@ -251,7 +232,7 @@ func writeFileCoverage(ctx context.Context, w io.Writer, repo string, ff *covera
 }
 
 func genFuncsCov(fc *coveragedb.FileCoverageWithLineInfo, ff *coveragedb.FunctionFinder,
-) ([]*functionCoverage, error) {
+) ([]*cover.FunctionCoverage, error) {
 	nameToLines := map[string][]int{}
 	funcInstrumented := map[string]int{}
 	for i, hitCount := range fc.HitCounts {
@@ -267,9 +248,9 @@ func genFuncsCov(fc *coveragedb.FileCoverageWithLineInfo, ff *coveragedb.Functio
 		nameToLines[funcName] = append(nameToLines[funcName], lineNum)
 	}
 
-	var res []*functionCoverage
+	var res []*cover.FunctionCoverage
 	for funcName, lines := range nameToLines {
-		res = append(res, &functionCoverage{
+		res = append(res, &cover.FunctionCoverage{
 			FuncName:     funcName,
 			Instrumented: funcInstrumented[funcName],
 			Blocks:       linesToBlocks(lines),
@@ -278,10 +259,10 @@ func genFuncsCov(fc *coveragedb.FileCoverageWithLineInfo, ff *coveragedb.Functio
 	return res, nil
 }
 
-func linesToBlocks(lines []int) []*coveredBlock {
-	var res []*coveredBlock
+func linesToBlocks(lines []int) []*cover.CoveredBlock {
+	var res []*cover.CoveredBlock
 	for _, lineNum := range lines {
-		res = append(res, &coveredBlock{
+		res = append(res, &cover.CoveredBlock{
 			FromLine: lineNum,
 			FromCol:  0,
 			ToLine:   lineNum,

dashboard/app/public_json_api_test.go

@@ -271,6 +271,7 @@ func TestWriteExtAPICoverageFor(t *testing.T) {
 			{
 				FileCoverageWithDetails: coveragedb.FileCoverageWithDetails{
 					Filepath: "/file",
+					Commit:   "test-commit",
 				},
 				LinesInstrumented: []int64{1, 2, 3},
 				HitCounts:         []int64{10, 20, 30},
@@ -283,7 +284,7 @@ func TestWriteExtAPICoverageFor(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, `{
 	"repo": "test-repo",
-	"commit": "",
+	"commit": "test-commit",
 	"file_path": "/file",
 	"functions": [
 		{

pkg/cover/backend/backend.go

@@ -14,8 +14,8 @@ import (
 type Impl struct {
 	Units           []*CompileUnit
 	Symbols         []*Symbol
-	Frames          []Frame
-	Symbolize       func(pcs map[*vminfo.KernelModule][]uint64) ([]Frame, error)
+	Frames          []*Frame
+	Symbolize       func(pcs map[*vminfo.KernelModule][]uint64) ([]*Frame, error)
 	CallbackPoints  []uint64
 	PreciseCoverage bool
 }

pkg/cover/backend/dwarf.go

@@ -243,7 +243,7 @@ func makeDWARFUnsafe(params *dwarfParams) (*Impl, error) {
 	impl := &Impl{
 		Units:   allUnits,
 		Symbols: allSymbols,
-		Symbolize: func(pcs map[*vminfo.KernelModule][]uint64) ([]Frame, error) {
+		Symbolize: func(pcs map[*vminfo.KernelModule][]uint64) ([]*Frame, error) {
 			return symbolize(target, &interner, objDir, srcDir, buildDir, splitBuildDelimiters, pcs)
 		},
 		CallbackPoints:  allCoverPoints[0],
@@ -402,7 +402,7 @@ func readTextRanges(debugInfo *dwarf.Data, module *vminfo.KernelModule, pcFix pc
 }
 
 func symbolizeModule(target *targets.Target, interner *symbolizer.Interner, objDir, srcDir, buildDir string,
-	splitBuildDelimiters []string, mod *vminfo.KernelModule, pcs []uint64) ([]Frame, error) {
+	splitBuildDelimiters []string, mod *vminfo.KernelModule, pcs []uint64) ([]*Frame, error) {
 	procs := min(runtime.GOMAXPROCS(0)/2, len(pcs)/1000)
 	const (
 		minProcs = 1
@@ -446,7 +446,7 @@ func symbolizeModule(target *targets.Target, interner *symbolizer.Interner, objD
 	}
 	close(pcchan)
 	var err0 error
-	var frames []Frame
+	var frames []*Frame
 	for p := 0; p < procs; p++ {
 		res := <-symbolizerC
 		if res.err != nil {
@@ -458,7 +458,7 @@ func symbolizeModule(target *targets.Target, interner *symbolizer.Interner, objD
 			if mod.Name != "" {
 				pc = frame.PC + mod.Addr
 			}
-			frames = append(frames, Frame{
+			frames = append(frames, &Frame{
 				Module:   mod,
 				PC:       pc,
 				Name:     interner.Do(name),
@@ -481,10 +481,10 @@ func symbolizeModule(target *targets.Target, interner *symbolizer.Interner, objD
 }
 
 func symbolize(target *targets.Target, interner *symbolizer.Interner, objDir, srcDir, buildDir string,
-	splitBuildDelimiters []string, pcs map[*vminfo.KernelModule][]uint64) ([]Frame, error) {
-	var frames []Frame
+	splitBuildDelimiters []string, pcs map[*vminfo.KernelModule][]uint64) ([]*Frame, error) {
+	var frames []*Frame
 	type frameResult struct {
-		frames []Frame
+		frames []*Frame
 		err    error
 	}
 	frameC := make(chan frameResult, len(pcs))

pkg/cover/backend/gvisor.go

@@ -54,7 +54,7 @@ func makeGvisor(target *targets.Target, objDir, srcDir, buildDir string, modules
 	return impl, nil
 }
 
-func gvisorSymbolize(bin, srcDir string) ([]Frame, error) {
+func gvisorSymbolize(bin, srcDir string) ([]*Frame, error) {
 	cmd := osutil.Command(bin, "symbolize", "-all")
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
@@ -66,7 +66,7 @@ func gvisorSymbolize(bin, srcDir string) ([]Frame, error) {
 	}
 	defer cmd.Wait()
 	defer cmd.Process.Kill()
-	var frames []Frame
+	var frames []*Frame
 	s := bufio.NewScanner(stdout)
 	for s.Scan() {
 		frame, err := gvisorParseLine(s)
@@ -93,27 +93,27 @@ func gvisorSymbolize(bin, srcDir string) ([]Frame, error) {
 	return frames, nil
 }
 
-func gvisorParseLine(s *bufio.Scanner) (Frame, error) {
+func gvisorParseLine(s *bufio.Scanner) (*Frame, error) {
 	pc, err := strconv.ParseUint(s.Text(), 0, 64)
 	if err != nil {
-		return Frame{}, fmt.Errorf("read pc %q, but no line info", pc)
+		return nil, fmt.Errorf("read pc %q, but no line info", pc)
 	}
 	if !s.Scan() {
-		return Frame{}, fmt.Errorf("read pc %q, but no line info", pc)
+		return nil, fmt.Errorf("read pc %q, but no line info", pc)
 	}
 	match := gvisorLineRe.FindStringSubmatch(s.Text())
 	if match == nil {
-		return Frame{}, fmt.Errorf("failed to parse line: %q", s.Text())
+		return nil, fmt.Errorf("failed to parse line: %q", s.Text())
 	}
 	var ints [4]int
 	for i := range ints {
 		x, err := strconv.ParseUint(match[i+3], 0, 32)
 		if err != nil {
-			return Frame{}, fmt.Errorf("failed to parse number %q: %w", match[i+3], err)
+			return nil, fmt.Errorf("failed to parse number %q: %w", match[i+3], err)
 		}
 		ints[i] = int(x)
 	}
-	frame := Frame{
+	frame := &Frame{
 		PC:   pc,
 		Name: match[2],
 		Range: Range{

pkg/cover/html.go

@@ -178,7 +178,7 @@ func fileLineContents(file *file, lines [][]byte) lineCoverExport {
 
 func (rg *ReportGenerator) DoRawCoverFiles(w io.Writer, params HandlerParams) error {
 	progs := fixUpPCs(params.Progs, params.Filter)
-	if err := rg.symbolizePCs(uniquePCs(progs)); err != nil {
+	if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil {
 		return err
 	}
 
@@ -223,7 +223,7 @@ func (rg *ReportGenerator) DoCoverJSONL(w io.Writer, params HandlerParams) error
 		}
 	}
 	progs := fixUpPCs(params.Progs, params.Filter)
-	if err := rg.symbolizePCs(uniquePCs(progs)); err != nil {
+	if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil {
 		return err
 	}
 	pcProgCount := make(map[uint64]int)
@@ -256,6 +256,96 @@ func (rg *ReportGenerator) DoCoverJSONL(w io.Writer, params HandlerParams) error
 	return nil
 }
 
+type CoveredBlock struct {
+	FromLine int `json:"from_line"`
+	FromCol  int `json:"from_column"`
+	ToLine   int `json:"to_line"`
+	ToCol    int `json:"to_column"`
+}
+
+type FunctionCoverage struct {
+	FuncName     string          `json:"func_name"`
+	Instrumented int             `json:"total_blocks,omitempty"`
+	Blocks       []*CoveredBlock `json:"covered_blocks"`
+}
+
+type FileCoverage struct {
+	Repo      string              `json:"repo,omitempty"`
+	Commit    string              `json:"commit,omitempty"`
+	FilePath  string              `json:"file_path"`
+	Functions []*FunctionCoverage `json:"functions"`
+}
+
+type ProgramCoverage struct {
+	Program      string          `json:"program"`
+	CoveredFiles []*FileCoverage `json:"coverage"`
+}
+
+// DoCoverPrograms returns the corpus programs with the associated coverage.
+// The result is a jsonl stream.
+// Each line is a single ProgramCoverage record.
+func (rg *ReportGenerator) DoCoverPrograms(w io.Writer, params HandlerParams) error {
+	if rg.CallbackPoints != nil {
+		if err := rg.symbolizePCs(rg.CallbackPoints); err != nil {
+			return fmt.Errorf("failed to symbolize PCs(): %w", err)
+		}
+	}
+	pcToFrames := map[uint64][]*backend.Frame{}
+	for _, frame := range rg.Frames {
+		pcToFrames[frame.PC] = append(pcToFrames[frame.PC], frame)
+	}
+	encoder := json.NewEncoder(w)
+	for _, prog := range params.Progs {
+		fileFuncFrames := map[string]map[string][]*backend.Frame{}
+		for _, pc := range uniquePCs(prog) {
+			for _, frame := range pcToFrames[pc] {
+				if fileFuncFrames[frame.Name] == nil {
+					fileFuncFrames[frame.Name] = map[string][]*backend.Frame{}
+				}
+				frames := fileFuncFrames[frame.Name][frame.FuncName]
+				frames = append(frames, frame)
+				fileFuncFrames[frame.Name][frame.FuncName] = frames
+			}
+		}
+
+		var progCoverage []*FileCoverage
+		for filePath, functions := range fileFuncFrames {
+			var expFuncs []*FunctionCoverage
+			for funcName, frames := range functions {
+				var expCoveredBlocks []*CoveredBlock
+				for _, frame := range frames {
+					endCol := frame.EndCol
+					if endCol == backend.LineEnd {
+						endCol = -1
+					}
+					expCoveredBlocks = append(expCoveredBlocks, &CoveredBlock{
+						FromCol:  frame.StartCol,
+						FromLine: frame.StartLine,
+						ToCol:    endCol,
+						ToLine:   frame.EndLine,
+					})
+				}
+				expFuncs = append(expFuncs, &FunctionCoverage{
+					FuncName: funcName,
+					Blocks:   expCoveredBlocks,
+				})
+			}
+			progCoverage = append(progCoverage, &FileCoverage{
+				FilePath:  filePath,
+				Functions: expFuncs,
+			})
+		}
+
+		if err := encoder.Encode(&ProgramCoverage{
+			Program:      prog.Data,
+			CoveredFiles: progCoverage,
+		}); err != nil {
+			return fmt.Errorf("encoder.Encode: %w", err)
+		}
+	}
+	return nil
+}
+
 func (rg *ReportGenerator) DoRawCover(w io.Writer, params HandlerParams) error {
 	progs := fixUpPCs(params.Progs, params.Filter)
 	var pcs []uint64

pkg/cover/report.go

@@ -81,7 +81,7 @@ type line struct {
 type fileMap map[string]*file
 
 func (rg *ReportGenerator) prepareFileMap(progs []Prog, force, debug bool) (fileMap, error) {
-	if err := rg.symbolizePCs(uniquePCs(progs)); err != nil {
+	if err := rg.symbolizePCs(uniquePCs(progs...)); err != nil {
 		return nil, err
 	}
 	files := make(fileMap)
@@ -108,7 +108,7 @@ func (rg *ReportGenerator) prepareFileMap(progs []Prog, force, debug bool) (file
 	}
 	matchedPC := false
 	for _, frame := range rg.Frames {
-		f := fileByFrame(files, &frame)
+		f := fileByFrame(files, frame)
 		ln := f.lines[frame.StartLine]
 		coveredBy := pcToProgs[frame.PC]
 		if len(coveredBy) == 0 {
@@ -187,7 +187,7 @@ func coverageCallbackMismatch(debug bool, numPCs int, unmatchedPCs map[uint64]bo
 		len(unmatchedPCs), numPCs, debugStr)
 }
 
-func uniquePCs(progs []Prog) []uint64 {
+func uniquePCs(progs ...Prog) []uint64 {
 	PCs := make(map[uint64]bool)
 	for _, p := range progs {
 		for _, pc := range p.PCs {