Updated config and radosgw

.kitchen.yml

@@ -27,8 +27,6 @@ suites:
     ceph:
       config:
         fsid: ae3f1d03-bacd-4a90-b869-1a4fabb107f2
-        mon_initial_members:
-          - "127.0.0.1"
 - name: osd
   run_list:
     - "role[ceph-osd]"

README.md

@@ -65,9 +65,9 @@ This cookbook can be used to implement a chosen cluster design. Most of the conf
 * `node['ceph']['config]'['global']['cluster network']` - a CIDR specification of a separate cluster replication network
 * `node['ceph']['config]'['global']['rgw dns name']` -  the main domain of the radosgw daemon
 
-Most notably, the configuration does **NOT** need to set the `mon_initial_members`, because the cookbook does a node search based on tags to find other mons in the same environment.
+Most notably, the configuration does **NOT** need to set the `mon initial members`, because the cookbook does a node search based on TAGS to find other mons in the same environment. However, you can add them to `node['ceph']['config']['global']['mon initial members'] = <whatever mon ip list you want>`
 
-The other set of attributes that this recipe needs is `node['ceph']['osd_devices']`, which is an array of OSD definitions, similar to the following:
+The other set of attributes that this recipe needs is `node['ceph']['osd']['devices']`, which is an array of OSD definitions, similar to the following:
 
 * {'device' => '/dev/sdb'} - Use a full disk for the OSD, with a small partition for the journal
 * {'type' => 'directory', 'device' => '/src/node/sdb1/ceph'} - Use a directory, and have a small file for the journal
@@ -149,12 +149,8 @@ Ceph RGW nodes should use the ceph-radosgw role
 ### Ceph RADOS Gateway (RGW)
 ### Note: Only supports the newer 'civetweb' version of RGW (not Apache)
 
-* `node['ceph']['radosgw']['api_fqdn']` - what vhost to configure in the web server
-* `node['ceph']['radosgw']['admin_email']` - the admin email address to configure in the web server
 * `node['ceph']['radosgw']['port']` - Port of the rgw. Defaults to 80
-* `node['ceph']['radosgw']['webserver_companion']` - defaults to 'apache2', but it can be set to 'civetweb', or to false in order to leave it unconfigured
-* `node['ceph']['radosgw']['path']` - where to save the s3gw.fcgi file
-* `node['ceph']['config']['global']['rgw dns name']` -  the main domain of the radosgw daemon, to calculate the bucket name from a subdomain
+* `node['ceph']['radosgw']['rgw_dns_name']` -  the main domain of the radosgw daemon, to calculate the bucket name from a subdomain
 
 ## Resources/Providers
 

attributes/default.rb

@@ -51,11 +51,8 @@
 
 # Override these in your environment file or here if you wish. Don't put them in the 'ceph''config''global' section.
 # The public and cluster network settings are critical for proper operations.
-default['ceph']['network']['public']['cidr'] = '10.0.101.0/24'
-default['ceph']['network']['cluster']['cidr'] = '192.168.101.0/24'
-
-# Will radosgw integrate with OpenStack Keystone - true/false
-default['ceph']['radosgw']['keystone_auth'] = false
+default['ceph']['network']['public']['cidr'] = ['10.121.1.0/24']
+default['ceph']['network']['cluster']['cidr'] = ['10.121.2.0/24']
 
 # Tags are used to identify nodes for searching.
 # IMPORTANT

attributes/mon.rb

@@ -16,8 +16,7 @@
 
 include_attribute 'ceph-chef'
 
-# The ceph mon ips attribute gets built in a wrapper recipe or chef-repo style environment like ceph-chef
-default['ceph']['mon']['ips'] = nil
+default['ceph']['mon']['port'] = 6789
 
 # init_style in each major section is allowed so that radosgw or osds or mons etc could be a different OS if required.
 # The default is everything on the same OS
@@ -28,9 +27,6 @@
 # MUST be set in the wrapper cookbook or chef-repo like project
 default['ceph']['mon']['role'] = 'search-ceph-mon'
 
-# Default of 15 seconds but change to nil for default of .050 or set it to .050
-default['ceph']['mon']['clock_drift_allowed'] = 15
-
 case node['platform_family']
 when 'debian', 'rhel', 'fedora'
   packages = ['ceph']

attributes/radosgw.rb

@@ -2,7 +2,7 @@
 # Cookbook Name:: ceph
 # Attributes:: radosgw
 #
-# Copyright 2015, Bloomberg Finance L.P.
+# Copyright 2016, Bloomberg Finance L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,8 +33,6 @@
 
 include_attribute 'ceph-chef'
 
-default['ceph']['radosgw']['api_fqdn'] = 'localhost'
-default['ceph']['radosgw']['admin_email'] = '[email protected]'
 default['ceph']['radosgw']['port'] = 80
 # default['ceph']['radosgw']['webserver'] = 'civetweb'
 # IMPORTANT: The civetweb user manual is a good place to look for custom config for civetweb:
@@ -46,11 +44,14 @@
 default['ceph']['radosgw']['civetweb_error_log_file'] = '/var/log/radosgw/civetweb.error.log'
 
 # OpenStack Keystone specific
-default['ceph']['radosgw']['keystone_admin_token'] = nil
-default['ceph']['radosgw']['keystone_url'] = nil
-default['ceph']['radosgw']['keystone_url_port'] = 35358
-
-default['ceph']['radosgw']['dns_name'] = nil
+# Will radosgw integrate with OpenStack Keystone - true/false
+default['ceph']['radosgw']['keystone']['auth'] = false
+default['ceph']['radosgw']['keystone']['admin']['token'] = nil
+default['ceph']['radosgw']['keystone']['admin']['url'] = nil
+default['ceph']['radosgw']['keystone']['admin']['port'] = 35357
+default['ceph']['radosgw']['keystone']['accepted_roles'] = 'admin Member _member_'
+default['ceph']['radosgw']['keystone']['token_cache_size'] = 1000
+default['ceph']['radosgw']['keystone']['revocation_interval'] = 1200
 
 # NOTE: For radosgw pools, see pools.rb attributes.
 

libraries/ceph_chef_helper.rb

@@ -287,17 +287,27 @@ def ceph_chef_save_restapi_secret(secret)
 # search for a matching monitor IP in the node environment.
 # 1. For each public network specified:
 #    a. We look if the network is IPv6 or IPv4
-#    b. We look for a route matching the network
+#    b. We look for a route matching the network. You can't assume all nodes will be part of the same subnet but they
+# MUST be part of the same aggregate subnet. For example, if you have 10.121.1.0/24 (class C) as your public IP block
+# and all of you racks/nodes are spanning the same CIDR block then all is well. However, if you have the same public IP
+# block and your racks/nodes are each routable (L3) then those racks/nodes MUST be part of the aggregate CIDR which is
+# 10.121.1.0/24 in the example here. So, you could have each rack of nodes on their own subnet like /27 which will give
+# you a max of 8 subnets under the aggregate of /24. For example, rack1 could be 10.121.1.0/27, rack2 - 10.121.1.32/27,
+# rack3 - 10.121.1.64/27 ... up to 8 racks in this example.
 #    c. If we found match, we return the IP with the port
-def ceph_chef_find_node_ip_in_network(network, nodeish = nil)
+# This function is important because we TAG nodes for specific roles and then search for those tags to dynamically
+# update the node data. Of course, another way would be to create node data specific to a given role such as mon, osd ...
+def ceph_chef_find_node_ip_in_network(networks, nodeish = nil)
   require 'netaddr'
   nodeish = node unless nodeish
-  network.split(/\s*,\s*/).each do |n|
-    net = NetAddr::CIDR.create(n)
-    nodeish['network']['interfaces'].each do |_iface, addrs|
-      addresses = addrs['addresses'] || []
-      addresses.each do |ip, params|
-        return ceph_chef_ip_address_to_ceph_chef_address(ip, params) if ceph_chef_ip_address_in_network?(ip, params, net)
+  networks.each do |network|
+    network.split(/\s*,\s*/).each do |n|
+      net = NetAddr::CIDR.create(n)
+      nodeish['network']['interfaces'].each do |_iface, addrs|
+        addresses = addrs['addresses'] || []
+        addresses.each do |ip, params|
+          return ceph_chef_ip_address_to_ceph_chef_address(ip, params) if ceph_chef_ip_address_in_network?(ip, params, net)
+        end
       end
     end
   end
@@ -319,19 +329,21 @@ def ceph_chef_ip_address_in_network?(ip, params, net)
   end
 end
 
+# To get subcidr blocks to work within a supercidr aggregate the logic has to change
+# from params['prefixlen'].to_i == net.bits to removing it
 def ceph_chef_ip4_address_in_network?(ip, params, net)
-  net.contains?(ip) && params.key?('broadcast') && params['prefixlen'].to_i == net.bits
+  net.contains?(ip) && params.key?('broadcast')
 end
 
 def ceph_chef_ip6_address_in_network?(ip, params, net)
-  net.contains?(ip) && params['prefixlen'].to_i == net.bits
+  net.contains?(ip) # && params['prefixlen'].to_i == net.bits
 end
 
 def ceph_chef_ip_address_to_ceph_chef_address(ip, params)
   if params['family'].eql?('inet')
-    return "#{ip}:6789"
+    return "#{ip}:#{node['ceph']['mon']['port']}"
   elsif params['family'].eql?('inet6')
-    return "[#{ip}]:6789"
+    return "[#{ip}]:#{node['ceph']['mon']['port']}"
   end
 end
 
@@ -425,15 +437,7 @@ def ceph_chef_mon_nodes_host(nodes)
 
 # Returns a list of ip:port of ceph mon for public network
 def ceph_chef_mon_addresses
-  # if File.exist?("/var/run/ceph/#{node['ceph']['cluster']}-mon.#{node['hostname']}.asok")
-  #   mon_ips = ceph_chef_quorum_members_ips
-  # else
-  #   if node['ceph']['mon']['ips']
-  #     mon_ips = node['ceph']['mon']['ips']
-  #   else
   mon_ips = ceph_chef_mon_nodes_ip(ceph_chef_mon_nodes)
-  #   end
-  # end
   mon_ips.reject { |m| m.nil? }.uniq
 end
 

metadata.rb

@@ -4,7 +4,7 @@
 license 'Apache v2.0'
 description 'Installs/Configures Ceph (Hammer and above)'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version '0.9.7'
+version '0.9.8'
 
 depends	'apache2', '>= 1.1.12'
 depends 'apt'

recipes/radosgw_users.rb

@@ -26,6 +26,7 @@
     rgw_admin_cap = JSON.parse(%x[radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['admin']['uid']}" --caps="users=*;buckets=*;metadata=*;usage=*;zone=*"])
   end
   not_if "radosgw-admin user info --uid='#{node['ceph']['radosgw']['user']['admin']['uid']}'"
+  ignore_failure true
 end
 
 # Create a test user unless you have overridden the attribute and removed the test user.
@@ -36,5 +37,6 @@
       rgw_tester_cap = JSON.parse(%x[radosgw-admin caps add --uid="#{node['ceph']['radosgw']['user']['test']['uid']}" --caps="#{node['ceph']['radosgw']['user']['test']['caps']}"])
     end
     not_if "radosgw-admin user info --uid='#{node['ceph']['radosgw']['user']['test']['uid']}'"
+    ignore_failure true
   end
 end

templates/default/ceph.conf.erb

@@ -20,15 +20,12 @@
   cephx sign messages = true
   # Note: mon host (required) and mon initial members (optional) should be in global section in addition
   # to the cluster and public network options since they are all critical to every node.
-  <% if node['ceph']['config']['mon_initial_members'] %>
-  mon initial members = <%= node['ceph']['config']['mon_initial_members'] %>
-  <% end -%>
   # List of all of the monitor nodes in the given cluster
   mon host = <%= @mon_addresses.sort.join(', ') %>
   # Suppress warning of too many pgs
   mon pg warn max per osd = 0
-  cluster network = <%= node['ceph']['network']['cluster']['cidr'] %>
-  public network = <%= node['ceph']['network']['public']['cidr'] %>
+  cluster network = <%= @node['ceph']['network']['cluster']['cidr'].join(',') %>
+  public network = <%= @node['ceph']['network']['public']['cidr'].join(',') %>
 <% if !node['ceph']['config']['global'].nil? -%>
   # This is very flexible section. You can add more options OR override options from above simply by
   # specifying the values in your wrapper cookbook or your "chef-repo". If you override values then
@@ -84,8 +81,6 @@
 [mon]
   host = <%= node['hostname'] %>
   <% if !node['ceph']['mon']['clock_drift_allowed'].nil? -%>
-  # Default is .050 out of box but 15 in ceph cookbook. VMs (used in development) often drift so set this in the environment file if desired to override the defaults
-  mon clock drift allowed = <%= node['ceph']['mon']['clock_drift_allowed'] %>
   <% end -%>
   <% if !node['ceph']['config']['mon'].nil? -%>
   <% node['ceph']['config']['mon'].sort.each do |k, v| %>
@@ -115,7 +110,7 @@
 <% if @is_admin -%>
 [client.admin]
   keyring = /etc/ceph/$cluster.client.admin.keyring
-  # Force the admin to have a socket 
+  # Force the admin to have a socket
   admin socket =
 <% end -%>
 
@@ -131,19 +126,17 @@
   log file = /var/log/radosgw/$cluster.client.radosgw.log
   # Increased to 1 to log HTTP return codes - http://tracker.ceph.com/issues/12432
   debug rgw = 1/0
-  <% if node['ceph']['radosgw']['keystone_auth'] %>
-  rgw keystone url = <%= node['ceph']['radosgw']['keystone_url'] %>:<%= node['ceph']['radosgw']['keystone_url_port'] %>
-  rgw keystone admin token = <%= node['ceph']['radosgw']['keystone_admin_token'] %>
-  rgw keystone accepted roles = admin Member _member_
-  rgw keystone token cache size = 1000
-  rgw keystone revocation interval = 1200
+  rgw dns name = <%= node['fqdn'] %>
+  <% if node['ceph']['radosgw']['keystone']['auth'] %>
+  rgw keystone url = <%= node['ceph']['radosgw']['keystone']['admin']['url'] %>:<%= node['ceph']['radosgw']['keystone']['admin']['port'] %>
+  rgw keystone admin token = <%= node['ceph']['radosgw']['keystone']['admin']['token'] %>
+  rgw keystone accepted roles = <%= node['ceph']['radosgw']['keystone']['accepted_roles'] %>
+  rgw keystone token cache size = <%= node['ceph']['radosgw']['keystone']['token_cache_size'] %>
+  rgw keystone revocation interval = <%= node['ceph']['radosgw']['keystone']['revocation_interval'] %>
   rgw s3 auth use keystone = true
   <% end %>
-  <% if !node['ceph']['radosgw']['dns_name'].nil? -%>
-  rgw dns name = <%= node['ceph']['radosgw']['dns_name'] %>
-  <% end %>
-  <% if !node['ceph']['config']['rgw'].nil? -%>
-  <% node['ceph']['config']['rgw'].sort.each do |k, v| %>
+  <% if !node['ceph']['config']['radosgw'].nil? -%>
+  <% node['ceph']['config']['radosgw'].sort.each do |k, v| %>
   <%= k %> = <%= v %>
   <% end %>
   <% end -%>