diff --git a/directory/directory.go b/directory/directory.go index 9e1bc643..51043a87 100644 --- a/directory/directory.go +++ b/directory/directory.go @@ -38,8 +38,9 @@ var ( // CompositeAnnotator wraps several annotators, and calls to Annotate() are forwarded to all of them. type CompositeAnnotator struct { - // latest date of the component annotators. This is precomputed, and returned by AnnotatorDate() - latestDate time.Time + // date of CompositeAnnotator is the earliest date of anntators inside this CA. + // It is precomputed, and returned by AnnotatorDate() + date time.Time annotators []api.Annotator } @@ -56,11 +57,19 @@ func (ca CompositeAnnotator) Annotate(ip string, ann *api.GeoData) error { return nil } +// PrintALl prints all dates inside this CompositeAnnotator +func (ca CompositeAnnotator) PrintAll() { + log.Println("Date of this CA: ", ca.date.Format("20160102")) + log.Println("contains anntators with the following dates:") + for i := range ca.annotators { + log.Println(ca.annotators[i].AnnotatorDate().Format("20160102")) + } +} // AnnotatorDate returns the date of the most recent wrapped annotator. Most recent is returned // as we try to apply the most recent annotators that predate the test we are annotating. So the // most recent of all the annotators is the date that should be compared to the test date. func (ca CompositeAnnotator) AnnotatorDate() time.Time { - return ca.latestDate + return ca.date } // Compute the latest AnnotatorDate() value from a slice of annotators. @@ -75,6 +84,17 @@ func computeLatestDate(annotators []api.Annotator) time.Time { return t } +func computerEarliestDate(annotators []api.Annotator) time.Time { + t := time.Now() + for i := range annotators { + at := annotators[i].AnnotatorDate() + if at.Before(t) { + t = at + } + } + return t +} + // String creates a string representation of the CA. // Base annotators will appear as [YYYYMMDD], and composite annotators as (A1A2), e.g., // ([20100102]([20110304][20120506])) @@ -95,7 +115,7 @@ func NewCompositeAnnotator(annotators []api.Annotator) api.Annotator { if annotators == nil { return nil } - ca := CompositeAnnotator{latestDate: computeLatestDate(annotators), annotators: annotators} + ca := CompositeAnnotator{date: computerEarliestDate(annotators), annotators: annotators} return ca } @@ -157,7 +177,7 @@ func advance(lists [][]api.Annotator) ([][]api.Annotator, bool) { // MergeAnnotators merges multiple lists of annotators, and returns a list of CompositeAnnotators. // Result will include a separate CompositeAnnotator for each unique date in any list, and each -// CA will include the most recent annotator from each list, prior to or equal to the CA date. +// CA will include annotator of different types, that was the earlist available one after the CA date. func MergeAnnotators(lists ...[]api.Annotator) []api.Annotator { listCount := len(lists) if listCount == 0 { @@ -229,3 +249,11 @@ func (d *Directory) lastEarlierThan(date time.Time) api.Annotator { } return d.annotators[index-1] } + +func (d *Directory) PrintAll() { + log.Println("Here are all datasets in dir currently:") + for _, ann := range d.annotators { + ann.(CompositeAnnotator).PrintAll() + } + log.Println("end of dir dataset list") +} diff --git a/directory/directory_test.go b/directory/directory_test.go index b445d4fb..ec8386e0 100644 --- a/directory/directory_test.go +++ b/directory/directory_test.go @@ -55,6 +55,7 @@ func TestBuild(t *testing.T) { }{ {"20170101", "20140608"}, {"20110101", "20100124"}, + {"20180501", "20180408"}, // TODO: Add test cases. } for _, tt := range tests { @@ -115,6 +116,15 @@ func TestMergeAnnotators(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := directory.MergeAnnotators(tt.lists[0], tt.lists[1]) // This is just a hack to allow us to create a useful signature. + expectedDate := make([]string, 3) + expectedDate[0] = "[20100101]" + expectedDate[1] = "[20100203]" + expectedDate[2] = "[20110101]" + for i, ann := range got { + if expectedDate[i] != ann.AnnotatorDate().Format("[20060102]") { + t.Errorf("Expect AnnotateDate %s got %s", expectedDate[i], ann.AnnotatorDate().Format("[20060102]")) + } + } gotString := directory.NewCompositeAnnotator(got).(directory.CompositeAnnotator).String() if gotString != tt.want { t.Errorf("MergeAnnotators() =\n %v want:\n %v", gotString, tt.want) diff --git a/manager/manager.go b/manager/manager.go index 133ae1cd..f833db09 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -188,18 +188,14 @@ func (bldr *listBuilder) build() []api.Annotator { // merge the legacy V4 & V6 annotators legacy := mergeV4V6(bldr.legacyV4.Fetch(), bldr.legacyV6.Fetch(), "legacy") - // Now append the Geolite2 annotators g2 := directory.SortSlice(bldr.geolite2.Fetch()) - geo := make([]api.Annotator, 0, len(g2)+len(legacy)) geo = append(geo, legacy...) geo = append(geo, g2...) - // here we have all the geo annotators in the ordered list. // now merge the ASN V4 & V6 annotators asn := mergeV4V6(bldr.asnV4.Fetch(), bldr.asnV6.Fetch(), "ASN") - // and now we need to create the composite annotators. First list is the // geo annotators, the second is the ASN combo := directory.MergeAnnotators(geo, asn) @@ -217,7 +213,6 @@ func (bldr *listBuilder) build() []api.Annotator { func mergeV4V6(v4Annotators, v6Annotators []api.Annotator, discriminator string) []api.Annotator { v4 := directory.SortSlice(v4Annotators) v6 := directory.SortSlice(v6Annotators) - var merged []api.Annotator if len(v4)*len(v6) < 1 { log.Printf("empty v4 or v6 annotator list for %s data, skipping", discriminator) diff --git a/manager/manager_test.go b/manager/manager_test.go index 006f2f91..7cba47b2 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -54,7 +54,7 @@ func TestInitDataset(t *testing.T) { `{"Geo":{"continent_code":"AS","country_code":"TH","country_code3":"THA","country_name":"Thailand","region":"40","city":"Bangkok","latitude":13.754,"longitude":100.501},"Network":{"Systems":[{"ASNs":[23969]}]}}`}, // This request needs a geolite2 dataset {"1.9.128.0", "1512086400", - `{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"05","city":"Pantai","latitude":2.787,"longitude":101.995},"Network":{"Systems":[{"ASNs":[4788]}]}}`}, + `{"Geo":{"continent_code":"AS","country_code":"MY","country_code3":"MYS","country_name":"Malaysia","region":"14","city":"Kuala Lumpur","postal_code":"50586","latitude":3.167,"longitude":101.7},"Network":{"Systems":[{"ASNs":[4788]}]}}`}, // This request needs the latest dataset in the memory. {"1.22.128.0", "1544400000", `{"Geo":{"continent_code":"AS","country_code":"IN","country_name":"India","region":"HR","city":"Faridabad","latitude":28.4333,"longitude":77.3167},"Network":{"Systems":[{"ASNs":[45528]}]}}`},