Skip to content

Commit

Permalink
Implements review comments, Adresses src-d#13
Browse files Browse the repository at this point in the history
  • Loading branch information
kyrcha committed Oct 4, 2019
1 parent 4070685 commit 59b778f
Show file tree
Hide file tree
Showing 10 changed files with 382 additions and 292 deletions.
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,5 @@
*.out
# Output of the go coverage tool
*.out

# .vscode workspace settings
.vscode
7 changes: 7 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
language: go
go_import_path: github.com/src-d/metdata-retrieval
go:
- 1.13.x
env:
global:
- SOURCED_GITHUB_TOKEN=$GITHUB_TOKEN
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -368,4 +368,12 @@ To test run:
GITHUB_TOKEN=<xxx> go test ./...
```

and

```
GITHUB_TOKEN=<xxx> go test -cover ./...
```

for coverage information.

Where `GITHUB_TOKEN` is a personal access token as described above.
296 changes: 122 additions & 174 deletions github/downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,204 +2,152 @@ package github

import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"testing"

"github.com/src-d/metadata-retrieval/github/store"
"github.com/stretchr/testify/require"
"github.com/src-d/metadata-retrieval/testutils"

"github.com/stretchr/testify/require"
"golang.org/x/oauth2"
)

func TestOnlineRepositoryDownload(t *testing.T) {
var tests = []struct {
owner string
repository string
version int
url string
createdAt string
isPrivate bool
isArchived bool
hasWiki bool
numOfPrs int
numOfPrComments int
}{
{
"git-fixtures",
"basic",
0,
"https://github.com/git-fixtures/basic",
"2015-03-31 11:42:21 +0000 UTC",
false,
false,
false,
2,
4,
},
{
"git-fixtures",
"symlinks",
0,
"https://github.com/git-fixtures/symlinks",
"2017-06-18 17:57:27 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"submodule",
0,
"https://github.com/git-fixtures/submodule",
"2017-02-12 17:18:40 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"releases",
0,
"https://github.com/git-fixtures/releases",
"2016-11-06 21:27:46 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"tags",
0,
"https://github.com/git-fixtures/tags",
"2016-09-21 19:15:12 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"empty",
0,
"https://github.com/git-fixtures/empty",
"2016-08-25 12:48:01 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"small-several-contributors",
0,
"https://github.com/git-fixtures/small-several-contributors",
"2016-02-23 18:58:59 +0000 UTC",
false,
false,
true,
0,
0,
},
{
"git-fixtures",
"small-several-contributors-with-prs",
0,
"https://github.com/git-fixtures/small-several-contributors-with-prs",
"2016-02-23 19:20:39 +0000 UTC",
false,
false,
true,
3,
1,
},
}
// RepositoryTest struct to hold a test oracle for a repository
type RepositoryTest struct {
Owner string `json:"owner"`
Repository string `json:"repository"`
Version int `json:"version"`
URL string `json:"url"`
CreatedAt string `json:"createdAt"`
IsPrivate bool `json:"isPrivate"`
IsArchived bool `json:"isArchived"`
HasWiki bool `json:"hasWiki"`
NumOfPrs int `json:"numOfPrs"`
NumOfPrComments int `json:"numOfPrComments"`
}

// OrganizationTest struct to hold a test oracle for an organization
type OrganizationTest struct {
Org string `json:"org"`
Version int `json:"version"`
URL string `json:"url"`
CreatedAt string `json:"createdAt"`
PublicRepos int `json:"publicRepos"`
TotalPrivateRepos int `json:"totalPrivateRepos"`
NumOfUsers int `json:"numOfUsers"`
}

client := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
))
// OnlineTests struct to hold the online tests
type OnlineTests struct {
RepositoryTests []RepositoryTest `json:"repositoryTests"`
OrganizationsTests []OrganizationTest `json:"organizationTests"`
}

var downloader, err = NewStdoutDownloader(client)
func checkToken(t *testing.T) {
if os.Getenv("GITHUB_TOKEN") == "" {
t.Skip("GITHUB_TOKEN is not set")
return
}
}

func loadOnlineTests(filepath string) (OnlineTests, error) {
var err error
var tests OnlineTests
jsonFile, err := os.Open(filepath)
if err != nil {
t.Errorf("Downloader failed to init with error:%s", err)
return tests, fmt.Errorf("Could not open json file: %v", err)
}
defer jsonFile.Close()
byteValue, _ := ioutil.ReadAll(jsonFile)
err = json.Unmarshal(byteValue, &tests)
if err != nil {
return tests, fmt.Errorf("Could not unmarshal json file: %v", err)
}
return tests, nil
}

saved := downloader.storer
defer func() { downloader.storer = saved }()
var teststore *store.TestStore = &store.TestStore{}
downloader.storer = teststore

for _, test := range tests {
err := downloader.DownloadRepository(context.TODO(), test.owner, test.repository, test.version)
require.Nil(t, err, "DownloadRepository(%s, %s) failed", test.owner, test.repository)
repository := teststore.GetRepository()
// Sample some properties that will not change, no topics available in git-fixtures
require.Equal(t, test.url, repository.Url, "DownloadRepository(%s, %s) Url %s not matching %s", test.owner, test.repository, repository.Url, test.url)
require.Equal(t, test.createdAt, repository.CreatedAt.String(), "DownloadRepository(%s, %s) CreatedAt %s not matching %s", test.owner, test.repository, test.createdAt, repository.CreatedAt.String())
require.Equal(t, test.isPrivate, repository.IsPrivate, "DownloadRepository(%s, %s) IsPrivate %s not matching %s", test.owner, test.repository, test.isPrivate, repository.IsPrivate)
require.Equal(t, test.isArchived, repository.IsArchived, "DownloadRepository(%s, %s) IsArchived %s not matching %s", test.owner, test.repository, test.isArchived, repository.IsArchived)
require.Equal(t, test.hasWiki, repository.HasWikiEnabled, "DownloadRepository(%s, %s) HasWikiEnabled %s not matching %s", test.owner, test.repository, test.hasWiki, repository.HasWikiEnabled)
require.Equal(t, test.numOfPrs, teststore.GetNumOfPrs(), "DownloadRepository(%s, %s) number of PRs %s not matching %s", test.owner, test.repository, test.numOfPrs, teststore.GetNumOfPrs())
require.Equal(t, test.numOfPrComments, teststore.GetNumOfPrComments(), "DownloadRepository(%s, %s) number of PR Comments %s not matching %s", test.owner, test.repository, test.numOfPrComments, teststore.GetNumOfPrComments())
func getDownloader() (*Downloader, *testutils.Memory, error) {
downloader, err := NewStdoutDownloader(
oauth2.NewClient(
context.TODO(),
oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
)))
if err != nil {
return nil, nil, err
}

storer := new(testutils.Memory)
downloader.storer = storer
return downloader, storer, nil
}

func TestOnlineOrganizationDownload(t *testing.T) {
var tests = []struct {
org string
version int
url string
createdAt string
publicRepos int
totalPrivateRepos int
numOfUsers int
}{
{
"git-fixtures",
0,
"https://github.com/git-fixtures",
"2016-02-23 18:57:27 +0000 UTC",
8,
0,
0,
},
func testOnlineRepo(t *testing.T, oracle RepositoryTest, d *Downloader, storer *testutils.Memory) {
err := d.DownloadRepository(context.TODO(), oracle.Owner, oracle.Repository, oracle.Version)
require := require.New(t) // Make a new require object for the specified test, so no need to pass it around
require.Nil(err)
// Sample some properties that will not change, no topics available in git-fixtures
require.Equal(oracle.URL, storer.Repository.Url)
require.Equal(oracle.CreatedAt, storer.Repository.CreatedAt.String())
require.Equal(oracle.IsPrivate, storer.Repository.IsPrivate)
require.Equal(oracle.IsArchived, storer.Repository.IsArchived)
require.Equal(oracle.HasWiki, storer.Repository.HasWikiEnabled)
require.Equal(oracle.NumOfPrs, len(storer.Prs))
require.Equal(oracle.NumOfPrComments, len(storer.PrComments))
}

// TestOnlineRepositoryDownload Tests the download of known and fixed GitHub repositories
func TestOnlineRepositoryDownload(t *testing.T) {
checkToken(t)
var err error
tests, err := loadOnlineTests("../testdata/online-repository-tests.json")

if err != nil {
t.Errorf("Failed to read the testcases:%s", err)
}

token := os.Getenv("GITHUB_TOKEN")
downloader, storer, err := getDownloader()
require.NoError(t, err)

client := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
))
for _, test := range tests.RepositoryTests {
t.Run(fmt.Sprintf("%s/%s", test.Owner, test.Repository), func(t *testing.T) {
testOnlineRepo(t, test, downloader, storer)
})
}
}

func testOnlineOrg(t *testing.T, oracle OrganizationTest, d *Downloader, storer *testutils.Memory) {
err := d.DownloadOrganization(context.TODO(), oracle.Org, oracle.Version)
require := require.New(t)
require.Nil(err, "DownloadOrganization(%s) failed", oracle.Org)
// Sample some properties that will not change, no topics available in git-fixtures
require.Equal(oracle.Org, storer.Organization.Name)
require.Equal(oracle.URL, storer.Organization.Url)
require.Equal(oracle.CreatedAt, storer.Organization.CreatedAt.String())
require.Equal(oracle.PublicRepos, storer.Organization.PublicRepos.TotalCount)
require.Equal(oracle.TotalPrivateRepos, storer.Organization.TotalPrivateRepos.TotalCount)
require.Equal(oracle.NumOfUsers, len(storer.Users))
}

var downloader, err = NewStdoutDownloader(client)
// TestOnlineOrganizationDownload Tests the download of known and fixed GitHub organization
func TestOnlineOrganizationDownload(t *testing.T) {
checkToken(t)
var err error
tests, err := loadOnlineTests("../testdata/online-organization-tests.json")

if err != nil {
t.Errorf("Downloader failed to init with error:%s", err)
t.Errorf("Failed to read the testcases:%s", err)
}

saved := downloader.storer
defer func() { downloader.storer = saved }()
var teststore *store.TestStore = &store.TestStore{}
downloader.storer = teststore

for _, test := range tests {
err := downloader.DownloadOrganization(context.TODO(), test.org, test.version)
require.Nil(t, err, "DownloadOrganization(%s) failed", test.org)
org := teststore.GetOrganization()
require.Equal(t, test.org, org.Name, "Organization name should be the same")
require.Equal(t, test.url, org.Url, "Organization url should be the same")
require.Equal(t, test.createdAt, org.CreatedAt.String(), "Organization created date should be the same")
require.Equal(t, test.publicRepos, org.PublicRepos.TotalCount, "Organization public repos count should be the same")
require.Equal(t, test.totalPrivateRepos, org.TotalPrivateRepos.TotalCount, "Organization total private repos count should be the same")
require.Equal(t, test.numOfUsers, teststore.GetNumOfUsers(), "Organization total private repos count should be the same")
downloader, storer, err := getDownloader()
require.NoError(t, err)

for _, test := range tests.OrganizationsTests {
t.Run(fmt.Sprintf("%s", test.Org), func(t *testing.T) {
testOnlineOrg(t, test, downloader, storer)
})
}

}
Loading

0 comments on commit 59b778f

Please sign in to comment.