merge with master

CHANGELOG.md

@@ -1,3 +1,36 @@
+## v0.5.10 - 2019-01-28
+
+  - Docs: Improve documentation regarding Alias (#868) @alexnederlof
+  - Adds a new flag `--aws-api-retries` which allows overriding the number of retries (#858) @viafoura
+  - Docs: Make awscli commands use JSON output (#849) @ifosch
+  - Docs: Add missing apiVersion to Ingress resource (#847) @shlao
+  - Fix for AWS private DNS zone (#844) @xianlubird
+  - Add support for AWS ELBs in eu-north-1 (#843) @argoyle
+  - Create a SECURITY_CONTACTS file (#842) @njuettner
+  - Use correct product name for Google Cloud DNS (#841) @seils
+  - Change default AWSBatchChangeSize to 1000 (#839) @medzin
+  - Fix dry-run mode in rfc2136 provider (#838) @lachlancooper
+  - Fix typos in rfc2136 provider (#837) @lachlancooper
+  - rfc2136 provider: one IP Target per RRSET (#836) @ivanfilippov
+  - Normalize DNS names during planning (#833) @justinsb
+  - Implement Stringer for planTableRow (#832) @justinsb
+  - Docs: Better security granularity concerning external dns service principal for Azure (#829) @DenisBiondic
+  - Docs: Update links in Cloudflare docs (#824) @PascalKu
+  - Docs: Add metrics info to FAQ (#822) @zachyam
+  - Docs: Update nameserver IPs in coredns.md (#820) @mozhuli
+  - Docs: Fix commands to cleanup Cloudflare (#818) @acrogenesis
+  - Avoid unnecessary updating for CRD resource (#810) @xunpan
+  - Fix issues with CoreDNS provider and more than 1 targets (#807) @xunpan
+  - AWS: Add zone tag filter (#804) @csrwng
+  - Docs: Update CoreDNS tutorial with RBAC manifest (#803) @Lujeni
+  - Use SOAP API to improve DYN's provider's performance (#799) @sanyu
+  - Expose managed resources and records as metrics (#793) @linki
+  - Docs: Updating Azure tutorial (#788) @pelithne
+  - Improve errors in Records() of Infoblox provider (#785) @dsbrng25b
+  - Change default apiVersion of CRD Source (#774) @dsbrng25b
+  - Allow setting Cloudflare proxying on a per-Ingress basis (#650) @eswets
+  - Support A record for multiple IPs for headless services (#645) @toshipp
+
 ## v0.5.9 - 2018-11-22
 
   - Core: Update delivery.yaml to new format (#782) @linki

Gopkg.toml

@@ -21,8 +21,8 @@ ignored = ["github.com/kubernetes/repo-infra/kazel"]
   version = "~1.13.7"
 
 [[constraint]]
+  branch = "master"
   name = "github.com/cloudflare/cloudflare-go"
-  version = "0.7.3"
 
 [[constraint]]
   name = "github.com/digitalocean/godo"

SECURITY_CONTACTS

@@ -10,7 +10,6 @@
 # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
 # INSTRUCTIONS AT https://kubernetes.io/security/
 
-linki
 njuettner
 hjacobs
 raffo

docs/contributing/crd-source.md

@@ -14,7 +14,11 @@ Here is typical example of [CRD API type](https://github.com/kubernetes-incubato
 ```go
 type TTL int64
 type Targets []string
-type ProviderSpecific map[string]string
+type ProviderSpecificProperty struct {
+	Name  string
+	Value string
+}
+type ProviderSpecific []ProviderSpecificProperty
 
 type Endpoint struct {
 	// The hostname of the DNS record

docs/contributing/crd-source/crd-manifest.yaml

@@ -33,7 +33,14 @@ spec:
                   labels:
                     type: object
                   providerSpecific:
-                    type: object
+                    items:
+                      properties:
+                        name: 
+                          type: string
+                        value:
+                          type: string
+                      type: object
+                    type: array
                   recordTTL:
                     format: int64
                     type: integer

docs/faq.md

@@ -243,3 +243,11 @@ To do this with ExternalDNS you can use the `--annotation-filter` to specificall
 an instance of a ingress controller. Let's assume you have two ingress controllers `nginx-internal` and `nginx-external`
 then you can start two ExternalDNS providers one with `--annotation-filter=kubernetes.io/ingress.class=nginx-internal`
 and one with `--annotation-filter=kubernetes.io/ingress.class=nginx-external`.
+
+### Can external-dns manage(add/remove) records in a hosted zone which is setup in different AWS account?
+
+Yes, give it the correct cross-account/assume-role permissions and use the `--aws-assume-role` flag https://github.com/kubernetes-incubator/external-dns/pull/524#issue-181256561
+
+### How do I provide multiple values to the annotation `external-dns.alpha.kubernetes.io/hostname`?
+
+Separate them by `,`.

docs/tutorials/aws.md

@@ -47,14 +47,14 @@ $ aws route53 create-hosted-zone --name "external-dns-test.my-org.com." --caller
 Make a note of the ID of the hosted zone you just created.
 
 ```console
-$ aws route53 list-hosted-zones-by-name --dns-name "external-dns-test.my-org.com." | jq -r '.HostedZones[0].Id'
+$ aws route53 list-hosted-zones-by-name --output json --dns-name "external-dns-test.my-org.com." | jq -r '.HostedZones[0].Id'
 /hostedzone/ZEWFWZ4R16P7IB
 ```
 
 Make a note of the nameservers that were assigned to your new zone.
 
 ```console
-$ aws route53 list-resource-record-sets --hosted-zone-id "/hostedzone/ZEWFWZ4R16P7IB" \
+$ aws route53 list-resource-record-sets --output json --hosted-zone-id "/hostedzone/ZEWFWZ4R16P7IB" \
     --query "ResourceRecordSets[?Type == 'NS']" | jq -r '.[0].ResourceRecords[].Value'
 ns-5514.awsdns-53.org.
 ...
@@ -177,7 +177,7 @@ Annotations which are specific to AWS.
 
 ### alias
 
-`external-dns.alpha.kubernetes.io/alias` if set to `true` on an ingress, it will create an ALIAS record when the target is an ALIAS as well.
+`external-dns.alpha.kubernetes.io/alias` if set to `true` on an ingress, it will create an ALIAS record when the target is an ALIAS as well. To make the target an alias, the ingress needs to be configured correctly as described in [the docs](./nginx-ingress.md#with-a-separate-tcp-load-balancer).
 
 ## Verify ExternalDNS works (Ingress example)
 
@@ -247,7 +247,7 @@ spec:
 After roughly two minutes check that a corresponding DNS record for your service was created.
 
 ```console
-$ aws route53 list-resource-record-sets --hosted-zone-id "/hostedzone/ZEWFWZ4R16P7IB" \
+$ aws route53 list-resource-record-sets --output json --hosted-zone-id "/hostedzone/ZEWFWZ4R16P7IB" \
     --query "ResourceRecordSets[?Name == 'nginx.external-dns-test.my-org.com.']|[?Type == 'A']"
 [
     {

docs/tutorials/cloudflare.md

@@ -196,3 +196,7 @@ Now that we have verified that ExternalDNS will automatically manage Cloudflare
 $ kubectl delete -f nginx.yaml
 $ kubectl delete -f externaldns.yaml
 ```
+
+## Setting cloudflare-proxied on a per-ingress basis
+
+Using the `external-dns.alpha.kubernetes.io/cloudflare-proxied: "true"` annotation on your ingress, you can specify if the proxy feature of Cloudflare should be enabled for that record. This setting will override the global `--cloudflare-proxied` setting.

endpoint/endpoint.go

@@ -109,8 +109,14 @@ func (t Targets) IsLess(o Targets) bool {
 	return false
 }
 
+// ProviderSpecificProperty holds the name and value of a configuration which is specific to individual DNS providers
+type ProviderSpecificProperty struct {
+	Name  string `json:"name,omitempty"`
+	Value string `json:"value,omitempty"`
+}
+
 // ProviderSpecific holds configuration which is specific to individual DNS providers
-type ProviderSpecific map[string]string
+type ProviderSpecific []ProviderSpecificProperty
 
 // Endpoint is a high-level way of a connection between a service and an IP
 type Endpoint struct {
@@ -160,10 +166,21 @@ func (e *Endpoint) WithProviderSpecific(key, value string) *Endpoint {
 	if e.ProviderSpecific == nil {
 		e.ProviderSpecific = ProviderSpecific{}
 	}
-	e.ProviderSpecific[key] = value
+
+	e.ProviderSpecific = append(e.ProviderSpecific, ProviderSpecificProperty{Name: key, Value: value})
 	return e
 }
 
+// GetProviderSpecificProperty returns a ProviderSpecificProperty if the property exists.
+func (e *Endpoint) GetProviderSpecificProperty(key string) (ProviderSpecificProperty, bool) {
+	for _, providerSpecific := range e.ProviderSpecific {
+		if providerSpecific.Name == key {
+			return providerSpecific, true
+		}
+	}
+	return ProviderSpecificProperty{}, false
+}
+
 func (e *Endpoint) String() string {
 	return fmt.Sprintf("%s %d IN %s %s %s", e.DNSName, e.RecordTTL, e.RecordType, e.Targets, e.ProviderSpecific)
 }

internal/testutils/endpoint.go

@@ -17,6 +17,7 @@ limitations under the License.
 package testutils
 
 import (
+	"reflect"
 	"sort"
 
 	"github.com/kubernetes-incubator/external-dns/endpoint"
@@ -49,7 +50,7 @@ func SameEndpoint(a, b *endpoint.Endpoint) bool {
 	return a.DNSName == b.DNSName && a.Targets.Same(b.Targets) && a.RecordType == b.RecordType &&
 		a.Labels[endpoint.OwnerLabelKey] == b.Labels[endpoint.OwnerLabelKey] && a.RecordTTL == b.RecordTTL &&
 		a.Labels[endpoint.ResourceLabelKey] == b.Labels[endpoint.ResourceLabelKey] &&
-		SameMap(a.ProviderSpecific, b.ProviderSpecific)
+		SameProverSpecific(a.ProviderSpecific, b.ProviderSpecific)
 }
 
 // SameEndpoints compares two slices of endpoints regardless of order
@@ -81,17 +82,7 @@ func SamePlanChanges(a, b map[string][]*endpoint.Endpoint) bool {
 		SameEndpoints(a["UpdateOld"], b["UpdateOld"]) && SameEndpoints(a["UpdateNew"], b["UpdateNew"])
 }
 
-// SameMap verifies that two maps contain the same string/string key/value pairs
-func SameMap(a, b map[string]string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-
-	for k, v := range a {
-		if v != b[k] {
-			return false
-		}
-	}
-
-	return true
+// SameProverSpecific verifies that two maps contain the same string/string key/value pairs
+func SameProverSpecific(a, b endpoint.ProviderSpecific) bool {
+	return reflect.DeepEqual(a, b)
 }

internal/testutils/endpoint_test.go

@@ -56,21 +56,23 @@ func ExampleSameEndpoints() {
 			RecordTTL:  endpoint.TTL(60),
 		},
 		{
-			DNSName:          "example.org",
-			Targets:          endpoint.Targets{"load-balancer.org"},
-			ProviderSpecific: endpoint.ProviderSpecific{"foo": "bar"},
+			DNSName: "example.org",
+			Targets: endpoint.Targets{"load-balancer.org"},
+			ProviderSpecific: endpoint.ProviderSpecific{
+				endpoint.ProviderSpecificProperty{Name: "foo", Value: "bar"},
+			},
 		},
 	}
 	sort.Sort(byAllFields(eps))
 	for _, ep := range eps {
 		fmt.Println(ep)
 	}
 	// Output:
-	// abc.com 0 IN A 1.2.3.4 map[]
-	// abc.com 0 IN TXT something map[]
-	// bbc.com 0 IN CNAME foo.com map[]
-	// cbc.com 60 IN CNAME foo.com map[]
-	// example.org 0 IN  load-balancer.org map[]
-	// example.org 0 IN  load-balancer.org map[foo:bar]
-	// example.org 0 IN TXT load-balancer.org map[]
+	// abc.com 0 IN A 1.2.3.4 []
+	// abc.com 0 IN TXT something []
+	// bbc.com 0 IN CNAME foo.com []
+	// cbc.com 60 IN CNAME foo.com []
+	// example.org 0 IN  load-balancer.org []
+	// example.org 0 IN  load-balancer.org [{foo bar}]
+	// example.org 0 IN TXT load-balancer.org []
 }

main.go

@@ -131,7 +131,7 @@ func main() {
 	case "azure":
 		p, err = provider.NewAzureProvider(cfg.AzureConfigFile, domainFilter, zoneIDFilter, cfg.AzureResourceGroup, cfg.DryRun)
 	case "cloudflare":
-		p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareProxied, cfg.DryRun)
+		p, err = provider.NewCloudFlareProvider(domainFilter, zoneIDFilter, cfg.CloudflareZonesPerPage, cfg.CloudflareProxied, cfg.DryRun)
 	case "google":
 		p, err = provider.NewGoogleProvider(cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.DryRun)
 	case "digitalocean":

pkg/apis/externaldns/types.go

@@ -66,6 +66,7 @@ type Config struct {
 	AzureConfigFile          string
 	AzureResourceGroup       string
 	CloudflareProxied        bool
+	CloudflareZonesPerPage   int
 	InfobloxGridHost         string
 	InfobloxWapiPort         int
 	InfobloxWapiUsername     string
@@ -140,6 +141,7 @@ var defaultConfig = &Config{
 	AzureConfigFile:          "/etc/kubernetes/azure.json",
 	AzureResourceGroup:       "",
 	CloudflareProxied:        false,
+	CloudflareZonesPerPage:   50,
 	InfobloxGridHost:         "",
 	InfobloxWapiPort:         443,
 	InfobloxWapiUsername:     "admin",
@@ -257,6 +259,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("azure-config-file", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure").Default(defaultConfig.AzureConfigFile).StringVar(&cfg.AzureConfigFile)
 	app.Flag("azure-resource-group", "When using the Azure provider, override the Azure resource group to use (optional)").Default(defaultConfig.AzureResourceGroup).StringVar(&cfg.AzureResourceGroup)
 	app.Flag("cloudflare-proxied", "When using the Cloudflare provider, specify if the proxy mode must be enabled (default: disabled)").BoolVar(&cfg.CloudflareProxied)
+	app.Flag("cloudflare-zones-per-page", "When using the Cloudflare provider, specify how many zones per page listed, max. possible 50 (default: 50)").Default(strconv.Itoa(defaultConfig.CloudflareZonesPerPage)).IntVar(&cfg.CloudflareZonesPerPage)
 	app.Flag("infoblox-grid-host", "When using the Infoblox provider, specify the Grid Manager host (required when --provider=infoblox)").Default(defaultConfig.InfobloxGridHost).StringVar(&cfg.InfobloxGridHost)
 	app.Flag("infoblox-wapi-port", "When using the Infoblox provider, specify the WAPI port (default: 443)").Default(strconv.Itoa(defaultConfig.InfobloxWapiPort)).IntVar(&cfg.InfobloxWapiPort)
 	app.Flag("infoblox-wapi-username", "When using the Infoblox provider, specify the WAPI username (default: admin)").Default(defaultConfig.InfobloxWapiUsername).StringVar(&cfg.InfobloxWapiUsername)

pkg/apis/externaldns/types_test.go

@@ -52,6 +52,7 @@ var (
 		AzureConfigFile:         "/etc/kubernetes/azure.json",
 		AzureResourceGroup:      "",
 		CloudflareProxied:       false,
+		CloudflareZonesPerPage:  50,
 		InfobloxGridHost:        "",
 		InfobloxWapiPort:        443,
 		InfobloxWapiUsername:    "admin",
@@ -88,8 +89,8 @@ var (
 		IstioIngressGateway:      "istio-other/istio-otheringressgateway",
 		Sources:                  []string{"service", "ingress", "connector"},
 		Namespace:                "namespace",
-		FQDNTemplate:             "{{.Name}}.service.example.com",
 		IgnoreHostnameAnnotation: true,
+		FQDNTemplate:             "{{.Name}}.service.example.com",
 		Compatibility:            "mate",
 		Provider:                 "google",
 		GoogleProject:            "project",
@@ -106,6 +107,7 @@ var (
 		AzureConfigFile:          "azure.json",
 		AzureResourceGroup:       "arg",
 		CloudflareProxied:        true,
+		CloudflareZonesPerPage:   20,
 		InfobloxGridHost:         "127.0.0.1",
 		InfobloxWapiPort:         8443,
 		InfobloxWapiUsername:     "infoblox",
@@ -175,6 +177,7 @@ func TestParseFlags(t *testing.T) {
 				"--azure-config-file=azure.json",
 				"--azure-resource-group=arg",
 				"--cloudflare-proxied",
+				"--cloudflare-zones-per-page=20",
 				"--infoblox-grid-host=127.0.0.1",
 				"--infoblox-wapi-port=8443",
 				"--infoblox-wapi-username=infoblox",
@@ -240,6 +243,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_AZURE_CONFIG_FILE":          "azure.json",
 				"EXTERNAL_DNS_AZURE_RESOURCE_GROUP":       "arg",
 				"EXTERNAL_DNS_CLOUDFLARE_PROXIED":         "1",
+				"EXTERNAL_DNS_CLOUDFLARE_ZONES_PER_PAGE":  "20",
 				"EXTERNAL_DNS_INFOBLOX_GRID_HOST":         "127.0.0.1",
 				"EXTERNAL_DNS_INFOBLOX_WAPI_PORT":         "8443",
 				"EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME":     "infoblox",

plan/plan.go

@@ -105,7 +105,7 @@ func (t planTable) getUpdates() (updateNew []*endpoint.Endpoint, updateOld []*en
 		if row.current != nil && len(row.candidates) > 0 { //dns name is taken
 			update := t.resolver.ResolveUpdate(row.current, row.candidates)
 			// compare "update" to "current" to figure out if actual update is required
-			if shouldUpdateTTL(update, row.current) || targetChanged(update, row.current) {
+			if shouldUpdateTTL(update, row.current) || targetChanged(update, row.current) || shouldUpdateProviderSpecific(update, row.current) {
 				inheritOwner(row.current, update)
 				updateNew = append(updateNew, update)
 				updateOld = append(updateOld, row.current)
@@ -185,6 +185,27 @@ func shouldUpdateTTL(desired, current *endpoint.Endpoint) bool {
 	return desired.RecordTTL != current.RecordTTL
 }
 
+func shouldUpdateProviderSpecific(desired, current *endpoint.Endpoint) bool {
+	if current.ProviderSpecific == nil && len(desired.ProviderSpecific) == 0 {
+		return false
+	}
+	for _, c := range current.ProviderSpecific {
+		// don't consider target health when detecting changes
+		// see: https://github.com/kubernetes-incubator/external-dns/issues/869#issuecomment-458576954
+		if c.Name == "aws/evaluate-target-health" {
+			continue
+		}
+
+		for _, d := range desired.ProviderSpecific {
+			if d.Name == c.Name && d.Value != c.Value {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // filterRecordsForPlan removes records that are not relevant to the planner.
 // Currently this just removes TXT records to prevent them from being
 // deleted erroneously by the planner (only the TXT registry should do this.)