Settings and public domain restructure (readthedocs#1829)

This performs some needed changes for readthedocs.com as well as for making the .org have knowledge of a secondary hosting domain:

Move settings to class based settings, it's far easier to manage with nested settings
Try to serve all docs (private or public) from the PUBLIC_DOMAIN, if it is set.
Remove old settings files that aren't used. Postgres settings were mostly our old production settings, that file shouldn't exist anymore. onebox.py was never used, and sqlite.py is basically just dev.py. For local postgres development, override DATABASES in local_settings.py, which should already be the case.
Settings for additional URL confs were added for override

manage.py

@@ -3,7 +3,7 @@
 import sys
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "readthedocs.settings.sqlite")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "readthedocs.settings.dev")
     sys.path.append(os.getcwd())
 
     from django.core.management import execute_from_command_line

readthedocs/api/base.py

@@ -46,7 +46,6 @@ def get_object_list(self, request):
         return super(ProjectResource, self).get_object_list(request)
 
     def dehydrate(self, bundle):
-        bundle.data['subdomain'] = "http://%s/" % bundle.obj.subdomain
         bundle.data['downloads'] = bundle.obj.get_downloads()
         return bundle
 
@@ -130,13 +129,6 @@ class Meta(object):
             "active": ALL,
         }
 
-    # Find a better name for this before including it.
-    # def dehydrate(self, bundle):
-    #     bundle.data['subdomain'] = "http://%s/en/%s/" % (
-    #         bundle.obj.project.subdomain, bundle.obj.slug
-    #     )
-    #     return bundle
-
     def get_object_list(self, request):
         self._meta.queryset = Version.objects.api(user=request.user)
         return super(VersionResource, self).get_object_list(request)

readthedocs/builds/models.py

@@ -182,7 +182,9 @@ def identifier_friendly(self):
 
     def get_subdomain_url(self):
         private = self.privacy_level == PRIVATE
-        return resolve(project=self.project, version_slug=self.slug, private=private)
+        return self.project.get_docs_url(version_slug=self.slug,
+                                         lang_slug=self.project.language,
+                                         private=private)
 
     def get_downloads(self, pretty=False):
         project = self.project

readthedocs/core/middleware.py

@@ -12,52 +12,72 @@
 log = logging.getLogger(__name__)
 
 LOG_TEMPLATE = u"(Middleware) {msg} [{host}{path}]"
+SUBDOMAIN_URLCONF = getattr(
+    settings,
+    'SUBDOMAIN_URLCONF',
+    'readthedocs.core.subdomain_urls'
+)
+SINGLE_VERSION_URLCONF = getattr(
+    settings,
+    'SINGLE_VERSION_URLCONF',
+    'readthedocs.core.single_version_urls'
+)
 
 
 class SubdomainMiddleware(object):
 
     def process_request(self, request):
+        if not getattr(settings, 'USE_SUBDOMAIN', False):
+            return None
+
         host = request.get_host().lower()
         path = request.get_full_path()
         log_kwargs = dict(host=host, path=path)
-        if settings.DEBUG:
-            log.debug(LOG_TEMPLATE.format(msg='DEBUG on, not processing middleware', **log_kwargs))
-            return None
+        public_domain = getattr(settings, 'PUBLIC_DOMAIN', None)
+        production_domain = getattr(settings, 'PRODUCTION_DOMAIN',
+                                    'readthedocs.org')
+
+        if public_domain is None:
+            public_domain = production_domain
         if ':' in host:
             host = host.split(':')[0]
         domain_parts = host.split('.')
 
         # Serve subdomains - but don't depend on the production domain only having 2 parts
-        if len(domain_parts) == len(settings.PRODUCTION_DOMAIN.split('.')) + 1:
+        if len(domain_parts) == len(public_domain.split('.')) + 1:
             subdomain = domain_parts[0]
             is_www = subdomain.lower() == 'www'
             is_ssl = subdomain.lower() == 'ssl'
-            if not is_www and not is_ssl and settings.PRODUCTION_DOMAIN in host:
+            if not is_www and not is_ssl and public_domain in host:
                 request.subdomain = True
                 request.slug = subdomain
-                request.urlconf = 'readthedocs.core.subdomain_urls'
+                request.urlconf = SUBDOMAIN_URLCONF
                 return None
         # Serve CNAMEs
-        if settings.PRODUCTION_DOMAIN not in host and \
-           'localhost' not in host and \
-           'testserver' not in host:
+        if (public_domain not in host and
+                production_domain not in host and
+                'localhost' not in host and
+                'testserver' not in host):
             request.cname = True
             domains = Domain.objects.filter(domain=host)
             if domains.count():
                 for domain in domains:
                     if domain.domain == host:
                         request.slug = domain.project.slug
-                        request.urlconf = 'core.subdomain_urls'
+                        request.urlconf = SUBDOMAIN_URLCONF
                         request.domain_object = True
                         log.debug(LOG_TEMPLATE.format(
-                            msg='Domain Object Detected: %s' % domain.domain, **log_kwargs))
+                            msg='Domain Object Detected: %s' % domain.domain,
+                            **log_kwargs))
                         break
-            if not hasattr(request, 'domain_object') and 'HTTP_X_RTD_SLUG' in request.META:
+            if (not hasattr(request, 'domain_object') and
+                    'HTTP_X_RTD_SLUG' in request.META):
                 request.slug = request.META['HTTP_X_RTD_SLUG'].lower()
-                request.urlconf = 'readthedocs.core.subdomain_urls'
+                request.urlconf = SUBDOMAIN_URLCONF
                 request.rtdheader = True
                 log.debug(LOG_TEMPLATE.format(
-                    msg='X-RTD-Slug header detetected: %s' % request.slug, **log_kwargs))
+                    msg='X-RTD-Slug header detetected: %s' % request.slug,
+                    **log_kwargs))
             # Try header first, then DNS
             elif not hasattr(request, 'domain_object'):
                 try:
@@ -73,7 +93,7 @@ def process_request(self, request):
                             msg='CNAME cached: %s->%s' % (slug, host),
                             **log_kwargs))
                     request.slug = slug
-                    request.urlconf = 'readthedocs.core.subdomain_urls'
+                    request.urlconf = SUBDOMAIN_URLCONF
                     log.debug(LOG_TEMPLATE.format(
                         msg='CNAME detetected: %s' % request.slug,
                         **log_kwargs))
@@ -134,9 +154,8 @@ def process_request(self, request):
                 # Let 404 be handled further up stack.
                 return None
 
-            if (getattr(proj, 'single_version', False) and
-                    not getattr(settings, 'USE_SUBDOMAIN', False)):
-                request.urlconf = 'readthedocs.core.single_version_urls'
+            if getattr(proj, 'single_version', False):
+                request.urlconf = SINGLE_VERSION_URLCONF
                 # Logging
                 host = request.get_host()
                 path = request.get_full_path()

readthedocs/core/resolver.py

@@ -29,9 +29,10 @@
     /docs/<project_slug>/projects/<subproject_slug>/<filename> # Subproject Single Version
 """
 
-from django.conf import settings
 import re
 
+from django.conf import settings
+
 from readthedocs.projects.constants import PRIVATE, PUBLIC
 
 
@@ -68,13 +69,14 @@ def _fix_filename(project, filename):
     return path
 
 
-def base_resolve_path(project_slug, filename, version_slug=None, language=None, private=False,
-                      single_version=None, subproject_slug=None,  subdomain=None, cname=None):
-    """ Resolve a with nothing smart, just filling in the blanks."""
-
-    if private:
-        url = '/docs/{project_slug}/'
-    elif subdomain or cname:
+def base_resolve_path(project_slug, filename, version_slug=None, language=None,
+                      private=False, single_version=None, subproject_slug=None,
+                      subdomain=None, cname=None):
+    """Resolve a with nothing smart, just filling in the blanks"""
+    # Only support `/docs/project' URLs outside our normal environment. Normally
+    # the path should always have a subdomain or CNAME domain
+    use_subdomain = getattr(settings, 'USE_SUBDOMAIN', False)
+    if subdomain or cname or (private and use_subdomain):
         url = '/'
     else:
         url = '/docs/{project_slug}/'
@@ -154,9 +156,7 @@ def resolve_domain(project, private=None):
 
     domain = canonical_project.domains.filter(canonical=True).first()
     # Force domain even if USE_SUBDOMAIN is on
-    if private:
-        return prod_domain
-    elif domain:
+    if domain:
         return domain.domain
     elif subdomain:
         subdomain_slug = canonical_project.slug.replace('_', '-')

readthedocs/core/settings.py

@@ -0,0 +1,25 @@
+"""Class based settings for complex settings inheritance"""
+
+import inspect
+import sys
+
+
+class Settings(object):
+
+    """Class-based settings wrapper"""
+
+    @classmethod
+    def load_settings(cls, module_name):
+        """Export class variables and properties to module namespace
+
+        This will export and class variable that is all upper case and doesn't
+        begin with ``_``. These members will be set as attributes on the module
+        ``module_name``.
+        """
+        self = cls()
+        module = sys.modules[module_name]
+        for (member, value) in inspect.getmembers(self):
+            if member.isupper() and not member.startswith('_'):
+                if isinstance(value, property):
+                    value = value.fget(self)
+                setattr(module, member, value)

readthedocs/core/symlink.py

@@ -60,8 +60,9 @@
 
 from django.conf import settings
 
+from readthedocs.builds.models import Version
 from readthedocs.projects import constants
-from readthedocs.projects.models import Domain, Project
+from readthedocs.projects.models import Domain
 from readthedocs.projects.utils import run
 
 log = logging.getLogger(__name__)
@@ -241,21 +242,20 @@ def symlink_single_version(self):
         Link from $WEB_ROOT/<project> ->
                   HOME/user_builds/<project>/rtd-builds/latest/
         """
-        default_version = self.get_default_version()
-        if default_version is None:
-            return
-
-        self._log("Symlinking single_version")
+        version = self.get_default_version()
 
+        # Clean up symlinks
         symlink = self.project_root
         if os.path.islink(symlink):
             os.unlink(symlink)
         if os.path.exists(symlink):
             shutil.rmtree(symlink)
 
-        # Where the actual docs live
-        docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', default_version)
-        run('ln -nsf %s/ %s' % (docs_dir, symlink))
+        # Create symlink
+        if version is not None:
+            docs_dir = os.path.join(settings.DOCROOT, self.project.slug,
+                                    'rtd-builds', version.slug)
+            run('ln -nsf %s/ %s' % (docs_dir, symlink))
 
     def symlink_versions(self):
         """Symlink project's versions
@@ -284,6 +284,14 @@ def symlink_versions(self):
                 if old_ver not in versions:
                     os.unlink(os.path.join(version_dir, old_ver))
 
+    def get_default_version(self):
+        """Look up project default version, return None if not found"""
+        default_version = self.project.get_default_version()
+        try:
+            return self.get_version_queryset().get(slug=default_version)
+        except Version.DoesNotExist:
+            return None
+
 
 class PublicSymlink(Symlink):
     CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root')
@@ -300,12 +308,6 @@ def get_subprojects(self):
     def get_translations(self):
         return self.project.translations.protected()
 
-    def get_default_version(self):
-        default_version = self.project.get_default_version()
-        if self.project.versions.protected().filter(slug=default_version).exists():
-            return default_version
-        return None
-
     def run_sanity_check(self):
         return self.project.privacy_level in [constants.PUBLIC, constants.PROTECTED]
 
@@ -327,10 +329,3 @@ def get_subprojects(self):
 
     def get_translations(self):
         return self.project.translations.private()
-
-    def get_default_version(self):
-        default_version = self.project.get_default_version()
-        version_qs = self.project.versions.private().filter(slug=default_version)
-        if version_qs.exists():
-            return default_version
-        return None

readthedocs/doc_builder/backends/mkdocs.py

@@ -105,7 +105,8 @@ def append_conf(self, **kwargs):
             'builder': "mkdocs",
             'docroot': docs_dir,
             'source_suffix': ".md",
-            'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
+            'api_host': getattr(settings, 'PUBLIC_API_URL',
+                                'https://readthedocs.org'),
             'commit': self.version.project.vcs_repo(self.version.slug).commit,
         }
         data_json = json.dumps(readthedocs_data, indent=4)

readthedocs/doc_builder/backends/sphinx.py

@@ -97,7 +97,8 @@ def append_conf(self, **kwargs):
                 'static_path': SPHINX_STATIC_DIR,
                 'template_path': SPHINX_TEMPLATE_DIR,
                 'conf_py_path': conf_py_path,
-                'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
+                'api_host': getattr(settings, 'PUBLIC_API_URL',
+                                    'https://readthedocs.org'),
                 # GitHub
                 'github_user': github_user,
                 'github_repo': github_repo,

readthedocs/projects/models.py

@@ -28,7 +28,7 @@
 from readthedocs.projects.utils import make_api_version, update_static_metadata
 from readthedocs.projects.version_handling import determine_stable_version
 from readthedocs.projects.version_handling import version_windows
-from readthedocs.core.resolver import resolve
+from readthedocs.core.resolver import resolve, resolve_domain
 from readthedocs.core.validators import validate_domain_name
 
 from readthedocs.vcs_support.base import VCSProject
@@ -290,16 +290,6 @@ class Meta:
     def __unicode__(self):
         return self.name
 
-    @property
-    def subdomain(self):
-        try:
-            domain = self.domains.get(canonical=True)
-            return domain.domain
-        except (Domain.DoesNotExist, MultipleObjectsReturned):
-            subdomain_slug = self.slug.replace('_', '-')
-            prod_domain = getattr(settings, 'PRODUCTION_DOMAIN')
-            return "%s.%s" % (subdomain_slug, prod_domain)
-
     def sync_supported_versions(self):
         supported = self.supported_versions()
         if supported:
@@ -421,6 +411,10 @@ def get_production_media_url(self, type_, version_slug, full_path=True):
             path = '//%s%s' % (settings.PRODUCTION_DOMAIN, path)
         return path
 
+    def subdomain(self):
+        """Get project subdomain from resolver"""
+        return resolve_domain(self)
+
     def get_downloads(self):
         downloads = {}
         downloads['htmlzip'] = self.get_production_media_url(

readthedocs/rtd_tests/mocks/mock_api.py

@@ -52,7 +52,6 @@ def get(self, **kwargs):
                     "requirements_file": "",
                     "resource_uri": "/api/v1/project/2599/",
                     "slug": "docs",
-                    "subdomain": "http://docs.readthedocs.org/",
                     "suffix": ".rst",
                     "theme": "default",
                     "install_project": false,

readthedocs/rtd_tests/tests/test_middleware.py

@@ -21,6 +21,7 @@ def create_user(username, password):
     return user
 
 
+@override_settings(USE_SUBDOMAIN=True)
 class MiddlewareTests(TestCase):
 
     def setUp(self):
@@ -57,7 +58,7 @@ def test_domain_object(self):
 
         request = self.factory.get(self.url, HTTP_HOST='docs.foobar.com')
         self.middleware.process_request(request)
-        self.assertEqual(request.urlconf, 'core.subdomain_urls')
+        self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls')
         self.assertEqual(request.domain_object, True)
         self.assertEqual(request.slug, 'pip')
 
@@ -100,8 +101,8 @@ def test_request_header_uppercase(self):
         self.assertEqual(request.rtdheader, True)
         self.assertEqual(request.slug, 'pip')
 
-    @override_settings(DEBUG=True)
-    def test_debug_on(self):
+    @override_settings(USE_SUBDOMAIN=True)
+    def test_use_subdomain_on(self):
         request = self.factory.get(self.url, HTTP_HOST='doesnt.really.matter')
         ret_val = self.middleware.process_request(request)
         self.assertEqual(ret_val, None)