diff --git a/404.html b/404.html
index 8df65ad92..ba05dcd4e 100644
--- a/404.html
+++ b/404.html
@@ -608,6 +608,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/data-plane-adoption/openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="/data-plane-adoption/openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/ceph/ceph_rbd/index.html b/ceph/ceph_rbd/index.html
index 3b7579e65..5b5d95f5f 100644
--- a/ceph/ceph_rbd/index.html
+++ b/ceph/ceph_rbd/index.html
@@ -619,6 +619,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/ceph/ceph_rgw/index.html b/ceph/ceph_rgw/index.html
index 75d110e92..31c3348dc 100644
--- a/ceph/ceph_rgw/index.html
+++ b/ceph/ceph_rgw/index.html
@@ -619,6 +619,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/contributing/development_environment/index.html b/contributing/development_environment/index.html
index 510997b46..579ef1548 100644
--- a/contributing/development_environment/index.html
+++ b/contributing/development_environment/index.html
@@ -619,6 +619,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/contributing/documentation/index.html b/contributing/documentation/index.html
index dcb617680..79ddf9658 100644
--- a/contributing/documentation/index.html
+++ b/contributing/documentation/index.html
@@ -619,6 +619,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/contributing/tests/index.html b/contributing/tests/index.html
index 9932fd926..83af7ef35 100644
--- a/contributing/tests/index.html
+++ b/contributing/tests/index.html
@@ -617,6 +617,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../../openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/index.html b/index.html
index c3f7edaf5..a53fbdd22 100644
--- a/index.html
+++ b/index.html
@@ -671,6 +671,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="openstack/heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="openstack/troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/backend_services_deployment/index.html b/openstack/backend_services_deployment/index.html
index cef8fcdd6..7a7795e70 100644
--- a/openstack/backend_services_deployment/index.html
+++ b/openstack/backend_services_deployment/index.html
@@ -696,6 +696,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
@@ -1035,6 +1055,8 @@ <h2 id="variables">Variables<a class="headerlink" href="#variables" title="Perma
 NOVA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep &#39; NovaPassword:&#39; | awk -F &#39;: &#39; &#39;{ print $2; }&#39;)
 OCTAVIA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep &#39; OctaviaPassword:&#39; | awk -F &#39;: &#39; &#39;{ print $2; }&#39;)
 PLACEMENT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep &#39; PlacementPassword:&#39; | awk -F &#39;: &#39; &#39;{ print $2; }&#39;)
+HEAT_AUTH_ENCRYPTION_KEY=$(cat ~/tripleo-standalone-passwords.yaml | grep &#39; HeatAuthEncryptionKey:&#39; | awk -F &#39;: &#39; &#39;{ print $2; }&#39;)
+HEAT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep &#39; HeatPassword:&#39; | awk -F &#39;: &#39; &#39;{ print $2; }&#39;)
 </code></pre></div>
 <h2 id="pre-checks">Pre-checks<a class="headerlink" href="#pre-checks" title="Permanent link">#</a></h2>
 <h2 id="procedure-backend-services-deployment">Procedure - backend services deployment<a class="headerlink" href="#procedure-backend-services-deployment" title="Permanent link">#</a></h2>
@@ -1070,6 +1092,8 @@ <h2 id="procedure-backend-services-deployment">Procedure - backend services depl
 oc set data secret/osp-secret &quot;NovaPassword=$NOVA_PASSWORD&quot;
 oc set data secret/osp-secret &quot;OctaviaPassword=$OCTAVIA_PASSWORD&quot;
 oc set data secret/osp-secret &quot;PlacementPassword=$PLACEMENT_PASSWORD&quot;
+oc set data secret/osp-secret &quot;HeatPassword=$HEAT_PASSWORD&quot;
+oc set data secret/osp-secret &quot;HeatAuthEncryptionKey=$HEAT_AUTH_ENCRYPTION_KEY&quot;
 </code></pre></div>
 <ul>
 <li>Deploy OpenStackControlPlane. <strong>Make sure to only enable DNS,
@@ -1208,7 +1232,7 @@ <h2 id="post-checks">Post-checks<a class="headerlink" href="#post-checks" title=
   <small>
     
       Last update:
-      2023-10-02
+      2023-10-12
     
   </small>
 </div>
diff --git a/openstack/ceph_backend_configuration/index.html b/openstack/ceph_backend_configuration/index.html
index 9d8aa697e..9948f2703 100644
--- a/openstack/ceph_backend_configuration/index.html
+++ b/openstack/ceph_backend_configuration/index.html
@@ -689,6 +689,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/cinder_adoption/index.html b/openstack/cinder_adoption/index.html
index 14651624d..49f8f70de 100644
--- a/openstack/cinder_adoption/index.html
+++ b/openstack/cinder_adoption/index.html
@@ -805,6 +805,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/edpm_adoption/index.html b/openstack/edpm_adoption/index.html
index 73c6d36f2..64f64df1b 100644
--- a/openstack/edpm_adoption/index.html
+++ b/openstack/edpm_adoption/index.html
@@ -696,6 +696,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/glance_adoption/index.html b/openstack/glance_adoption/index.html
index 0a4006c04..1ee62348d 100644
--- a/openstack/glance_adoption/index.html
+++ b/openstack/glance_adoption/index.html
@@ -722,6 +722,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/heat_adoption/index.html b/openstack/heat_adoption/index.html
new file mode 100644
index 000000000..9cfa49ed7
--- /dev/null
+++ b/openstack/heat_adoption/index.html
@@ -0,0 +1,1234 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+        <meta name="description" content="OpenStack Data Plane Adoption">
+      
+      
+        <meta name="author" content="OpenStack Team">
+      
+      
+        <link rel="canonical" href="https://openstack-k8s-operators.github.io/data-plane-adoption/openstack/heat_adoption/">
+      
+      
+        <link rel="prev" href="../ironic_adoption/">
+      
+      
+        <link rel="next" href="../troubleshooting/">
+      
+      
+      <link rel="icon" href="../../assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.4.7">
+    
+    
+      
+        <title>Heat adoption - OpenStack Data Plane Adoption</title>
+      
+    
+    
+      <link rel="stylesheet" href="../../assets/stylesheets/main.4b4a2bd9.min.css">
+      
+        
+        <link rel="stylesheet" href="../../assets/stylesheets/palette.356b1318.min.css">
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="teal" data-md-color-accent="indigo">
+  
+    
+    
+      <script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#heat-adoption" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+      <div data-md-color-scheme="default" data-md-component="outdated" hidden>
+        
+      </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="../.." title="OpenStack Data Plane Adoption" class="md-header__button md-logo" aria-label="OpenStack Data Plane Adoption" data-md-component="logo">
+      
+  <img src="../../images/openstack-logo.png" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenStack Data Plane Adoption
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              Heat adoption
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+        <form class="md-header__option" data-md-component="palette">
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="teal" data-md-color-accent="indigo"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
+    
+      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="teal" data-md-color-accent="indigo"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
+    
+      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
+      </label>
+    
+  
+</form>
+      
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+          <a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
+            
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7 0-.24-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91 1.61 0 2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08Z"/></svg>
+          </a>
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/openstack-k8s-operators/data-plane-adoption" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    openstack-k8s-operators/data-plane-adoption
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+          
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="../.." title="OpenStack Data Plane Adoption" class="md-nav__button md-logo" aria-label="OpenStack Data Plane Adoption" data-md-component="logo">
+      
+  <img src="../../images/openstack-logo.png" alt="logo">
+
+    </a>
+    OpenStack Data Plane Adoption
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/openstack-k8s-operators/data-plane-adoption" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81z"/></svg>
+  </div>
+  <div class="md-source__repository">
+    openstack-k8s-operators/data-plane-adoption
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../.." class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Overview
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
+        
+          
+          <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    OpenStack
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_2">
+            <span class="md-nav__icon md-icon"></span>
+            OpenStack
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../planning/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Planning the new deployment
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../backend_services_deployment/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Backend services deployment
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pull_openstack_configuration/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Pull Openstack configuration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../stop_openstack_services/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Stop OpenStack services
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../mariadb_copy/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    MariaDB data copy
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ovn_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    OVN data migration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../keystone_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Keystone adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../neutron_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Neutron adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ceph_backend_configuration/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Ceph backend configuration (if applicable)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../glance_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Glance adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../placement_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Placement adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../cinder_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Cinder adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../horizon_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Horizon adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../edpm_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    EDPM adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ironic_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Ironic adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+    
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    Prerequisites
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#procedure-heat-adoption" class="md-nav__link">
+    Procedure - Heat adoption
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#post-checks" class="md-nav__link">
+    Post-checks
+  </a>
+  
+    <nav class="md-nav" aria-label="Post-checks">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#check-that-heat-service-is-registered-in-keystone" class="md-nav__link">
+    Check that Heat service is registered in Keystone
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#check-heat-engine-services-are-up" class="md-nav__link">
+    Check Heat engine services are up
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#verify-you-can-now-see-your-heat-stacks-again" class="md-nav__link">
+    Verify you can now see your Heat stacks again
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../troubleshooting/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Troubleshooting
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
+        
+          
+          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Ceph
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            Ceph
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../ceph/ceph_rbd/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Ceph RBD migration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../ceph/ceph_rgw/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Ceph RGW migration
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+    
+    
+    
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
+        
+          
+          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Contributing
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4">
+            <span class="md-nav__icon md-icon"></span>
+            Contributing
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../contributing/documentation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Contributing to documentation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../contributing/development_environment/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Development environment
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../../contributing/tests/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tests
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    Prerequisites
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#procedure-heat-adoption" class="md-nav__link">
+    Procedure - Heat adoption
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#post-checks" class="md-nav__link">
+    Post-checks
+  </a>
+  
+    <nav class="md-nav" aria-label="Post-checks">
+      <ul class="md-nav__list">
+        
+          <li class="md-nav__item">
+  <a href="#check-that-heat-service-is-registered-in-keystone" class="md-nav__link">
+    Check that Heat service is registered in Keystone
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#check-heat-engine-services-are-up" class="md-nav__link">
+    Check Heat engine services are up
+  </a>
+  
+</li>
+        
+          <li class="md-nav__item">
+  <a href="#verify-you-can-now-see-your-heat-stacks-again" class="md-nav__link">
+    Verify you can now see your Heat stacks again
+  </a>
+  
+</li>
+        
+      </ul>
+    </nav>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+  
+
+
+<h1 id="heat-adoption">Heat adoption<a class="headerlink" href="#heat-adoption" title="Permanent link">#</a></h1>
+<p>Adopting Heat means that an existing <code>OpenStackControlPlane</code> CR, where Heat
+is supposed to be disabled, should be patched to start the service with the
+configuration parameters provided by the source environment.</p>
+<p>After the adoption process has been completed, a user can expect that they
+will then have CR's for <code>Heat</code>, <code>HeatAPI</code>, <code>HeatEngine</code> and <code>HeatCFNAPI</code>.
+Additionally, a user should have endpoints created within Keystone to facilitate
+the above mentioned servies.</p>
+<p>This guide also assumes that:</p>
+<ol>
+<li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li>
+<li>A OpenShift environment is running on the other side.</li>
+</ol>
+<h2 id="prerequisites">Prerequisites<a class="headerlink" href="#prerequisites" title="Permanent link">#</a></h2>
+<ul>
+<li>Previous Adoption steps completed. Notably, MariaDB and Keystone
+  should be already adopted.</li>
+<li>In addition, if your existing Heat stacks contain resources from other services
+  such as Neutron, Nova, Swift, etc. Those services should be adopted first before
+  trying to adopt Heat.</li>
+</ul>
+<h2 id="procedure-heat-adoption">Procedure - Heat adoption<a class="headerlink" href="#procedure-heat-adoption" title="Permanent link">#</a></h2>
+<p>As already done for <a href="https://github.com/openstack-k8s-operators/data-plane-adoption/blob/main/keystone_adoption.md">Keystone</a>, the Heat Adoption follows a similar pattern.</p>
+<p>Patch the <code>osp-secret</code> to update the <code>HeatAuthEncryptionKey</code> and <code>HeatPassword</code>. This needs
+to match what you have configured in the existing TripleO Heat configuration.</p>
+<p>You can retrieve and verify the existing <code>auth_encryption_key</code> and <code>service</code> passwords via:
+<div class="highlight"><pre><span></span><code><span class="o">[</span>stack@rhosp17<span class="w"> </span>~<span class="o">]</span>$<span class="w"> </span>grep<span class="w"> </span>-E<span class="w"> </span><span class="s1">&#39;HeatPassword|HeatAuth&#39;</span><span class="w"> </span>~/overcloud-deploy/overcloud/overcloud-passwords.yaml
+<span class="w">  </span>HeatAuthEncryptionKey:<span class="w"> </span>Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2
+<span class="w">  </span>HeatPassword:<span class="w"> </span>dU2N0Vr2bdelYH7eQonAwPfI3
+</code></pre></div></p>
+<p>And verifying on one of the Controllers that this is indeed the value in use:
+<div class="highlight"><pre><span></span><code><span class="o">[</span>stack@rhosp17<span class="w"> </span>~<span class="o">]</span>$<span class="w"> </span>ansible<span class="w"> </span>-i<span class="w"> </span>overcloud-deploy/overcloud/config-download/overcloud/tripleo-ansible-inventory.yaml<span class="w"> </span>overcloud-controller-0<span class="w"> </span>-m<span class="w"> </span>shell<span class="w"> </span>-a<span class="w"> </span><span class="s2">&quot;grep auth_encryption_key /var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf | grep -Ev &#39;^#|^</span>$<span class="s2">&#39;&quot;</span><span class="w"> </span>-b
+overcloud-controller-0<span class="w"> </span><span class="p">|</span><span class="w"> </span>CHANGED<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="nv">rc</span><span class="o">=</span><span class="m">0</span><span class="w"> </span>&gt;&gt;
+<span class="nv">auth_encryption_key</span><span class="o">=</span>Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2
+</code></pre></div></p>
+<p>This password needs to be base64 encoded and added to the <code>osp-secret</code>
+<div class="highlight"><pre><span></span><code>❯<span class="w"> </span><span class="nb">echo</span><span class="w"> </span>Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2<span class="w"> </span><span class="p">|</span><span class="w"> </span>base64
+UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK
+
+❯<span class="w"> </span>oc<span class="w"> </span>patch<span class="w"> </span>secret<span class="w"> </span>osp-secret<span class="w"> </span>--type<span class="o">=</span><span class="s1">&#39;json&#39;</span><span class="w"> </span>-p<span class="o">=</span><span class="s1">&#39;[{&quot;op&quot; : &quot;replace&quot; ,&quot;path&quot; : &quot;/data/HeatAuthEncryptionKey&quot; ,&quot;value&quot; : &quot;UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK&quot;}]&#39;</span>
+secret/osp-secret<span class="w"> </span>patched
+</code></pre></div></p>
+<p>Patch OpenStackControlPlane to deploy Heat:</p>
+<div class="highlight"><pre><span></span><code>oc<span class="w"> </span>patch<span class="w"> </span>openstackcontrolplane<span class="w"> </span>openstack<span class="w"> </span>--type<span class="o">=</span>merge<span class="w"> </span>--patch<span class="w"> </span><span class="s1">&#39;</span>
+<span class="s1">spec:</span>
+<span class="s1">  heat:</span>
+<span class="s1">    enabled: true</span>
+<span class="s1">    apiOverride:</span>
+<span class="s1">      route: {}</span>
+<span class="s1">    template:</span>
+<span class="s1">      databaseInstance: openstack</span>
+<span class="s1">      secret: osp-secret</span>
+<span class="s1">      memcachedInstance: memcached</span>
+<span class="s1">      passwordSelectors:</span>
+<span class="s1">        authEncryptionKey: HeatAuthEncryptionKey</span>
+<span class="s1">        database: HeatDatabasePassword</span>
+<span class="s1">        service: HeatPassword</span>
+<span class="s1">&#39;</span>
+</code></pre></div>
+<h2 id="post-checks">Post-checks<a class="headerlink" href="#post-checks" title="Permanent link">#</a></h2>
+<p>Ensure all of the CR's reach the "Setup Complete" state:
+<div class="highlight"><pre><span></span><code>❯<span class="w"> </span>oc<span class="w"> </span>get<span class="w"> </span>Heat,HeatAPI,HeatEngine,HeatCFNAPI
+NAME<span class="w">                           </span>STATUS<span class="w">   </span>MESSAGE
+heat.heat.openstack.org/heat<span class="w">   </span>True<span class="w">     </span>Setup<span class="w"> </span><span class="nb">complete</span>
+
+NAME<span class="w">                                  </span>STATUS<span class="w">   </span>MESSAGE
+heatapi.heat.openstack.org/heat-api<span class="w">   </span>True<span class="w">     </span>Setup<span class="w"> </span><span class="nb">complete</span>
+
+NAME<span class="w">                                        </span>STATUS<span class="w">   </span>MESSAGE
+heatengine.heat.openstack.org/heat-engine<span class="w">   </span>True<span class="w">     </span>Setup<span class="w"> </span><span class="nb">complete</span>
+
+NAME<span class="w">                                        </span>STATUS<span class="w">   </span>MESSAGE
+heatcfnapi.heat.openstack.org/heat-cfnapi<span class="w">   </span>True<span class="w">     </span>Setup<span class="w"> </span><span class="nb">complete</span>
+</code></pre></div></p>
+<h3 id="check-that-heat-service-is-registered-in-keystone">Check that Heat service is registered in Keystone<a class="headerlink" href="#check-that-heat-service-is-registered-in-keystone" title="Permanent link">#</a></h3>
+<div class="highlight"><pre><span></span><code><span class="w"> </span>oc<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-it<span class="w"> </span>openstackclient<span class="w"> </span>--<span class="w"> </span>openstack<span class="w"> </span>service<span class="w"> </span>list<span class="w"> </span>-c<span class="w"> </span>Name<span class="w"> </span>-c<span class="w"> </span>Type
++------------+----------------+
+<span class="p">|</span><span class="w"> </span>Name<span class="w">       </span><span class="p">|</span><span class="w"> </span>Type<span class="w">           </span><span class="p">|</span>
++------------+----------------+
+<span class="p">|</span><span class="w"> </span>heat<span class="w">       </span><span class="p">|</span><span class="w"> </span>orchestration<span class="w">  </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>glance<span class="w">     </span><span class="p">|</span><span class="w"> </span>image<span class="w">          </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>heat-cfn<span class="w">   </span><span class="p">|</span><span class="w"> </span>cloudformation<span class="w"> </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>ceilometer<span class="w"> </span><span class="p">|</span><span class="w"> </span>Ceilometer<span class="w">     </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>keystone<span class="w">   </span><span class="p">|</span><span class="w"> </span>identity<span class="w">       </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>placement<span class="w">  </span><span class="p">|</span><span class="w"> </span>placement<span class="w">      </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>cinderv3<span class="w">   </span><span class="p">|</span><span class="w"> </span>volumev3<span class="w">       </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>nova<span class="w">       </span><span class="p">|</span><span class="w"> </span>compute<span class="w">        </span><span class="p">|</span>
+<span class="p">|</span><span class="w"> </span>neutron<span class="w">    </span><span class="p">|</span><span class="w"> </span>network<span class="w">        </span><span class="p">|</span>
++------------+----------------+
+</code></pre></div>
+<div class="highlight"><pre><span></span><code>❯<span class="w"> </span>oc<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-it<span class="w"> </span>openstackclient<span class="w"> </span>--<span class="w"> </span>openstack<span class="w"> </span>endpoint<span class="w"> </span>list<span class="w"> </span>--service<span class="o">=</span>heat<span class="w"> </span>-f<span class="w"> </span>yaml
+-<span class="w"> </span>Enabled:<span class="w"> </span><span class="nb">true</span>
+<span class="w">  </span>ID:<span class="w"> </span>1da7df5b25b94d1cae85e3ad736b25a5
+<span class="w">  </span>Interface:<span class="w"> </span>public
+<span class="w">  </span>Region:<span class="w"> </span>regionOne
+<span class="w">  </span>Service<span class="w"> </span>Name:<span class="w"> </span>heat
+<span class="w">  </span>Service<span class="w"> </span>Type:<span class="w"> </span>orchestration
+<span class="w">  </span>URL:<span class="w"> </span>http://heat-api-public-openstack-operators.apps.okd.bne-shift.net/v1/%<span class="o">(</span>tenant_id<span class="o">)</span>s
+-<span class="w"> </span>Enabled:<span class="w"> </span><span class="nb">true</span>
+<span class="w">  </span>ID:<span class="w"> </span>414dd03d8e9d462988113ea0e3a330b0
+<span class="w">  </span>Interface:<span class="w"> </span>internal
+<span class="w">  </span>Region:<span class="w"> </span>regionOne
+<span class="w">  </span>Service<span class="w"> </span>Name:<span class="w"> </span>heat
+<span class="w">  </span>Service<span class="w"> </span>Type:<span class="w"> </span>orchestration
+<span class="w">  </span>URL:<span class="w"> </span>http://heat-api-internal.openstack-operators.svc:8004/v1/%<span class="o">(</span>tenant_id<span class="o">)</span>s
+</code></pre></div>
+<h3 id="check-heat-engine-services-are-up">Check Heat engine services are up<a class="headerlink" href="#check-heat-engine-services-are-up" title="Permanent link">#</a></h3>
+<div class="highlight"><pre><span></span><code><span class="w"> </span>oc<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-it<span class="w"> </span>openstackclient<span class="w"> </span>--<span class="w"> </span>openstack<span class="w"> </span>orchestration<span class="w"> </span>service<span class="w"> </span>list<span class="w"> </span>-f<span class="w"> </span>yaml
+-<span class="w"> </span>Binary:<span class="w"> </span>heat-engine
+<span class="w">  </span>Engine<span class="w"> </span>ID:<span class="w"> </span>b16ad899-815a-4b0c-9f2e-e6d9c74aa200
+<span class="w">  </span>Host:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Hostname:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Status:<span class="w"> </span>up
+<span class="w">  </span>Topic:<span class="w"> </span>engine
+<span class="w">  </span>Updated<span class="w"> </span>At:<span class="w"> </span><span class="s1">&#39;2023-10-11T21:48:01.000000&#39;</span>
+-<span class="w"> </span>Binary:<span class="w"> </span>heat-engine
+<span class="w">  </span>Engine<span class="w"> </span>ID:<span class="w"> </span>887ed392-0799-4310-b95c-ac2d3e6f965f
+<span class="w">  </span>Host:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Hostname:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Status:<span class="w"> </span>up
+<span class="w">  </span>Topic:<span class="w"> </span>engine
+<span class="w">  </span>Updated<span class="w"> </span>At:<span class="w"> </span><span class="s1">&#39;2023-10-11T21:48:00.000000&#39;</span>
+-<span class="w"> </span>Binary:<span class="w"> </span>heat-engine
+<span class="w">  </span>Engine<span class="w"> </span>ID:<span class="w"> </span>26ed9668-b3f2-48aa-92e8-2862252485ea
+<span class="w">  </span>Host:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Hostname:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Status:<span class="w"> </span>up
+<span class="w">  </span>Topic:<span class="w"> </span>engine
+<span class="w">  </span>Updated<span class="w"> </span>At:<span class="w"> </span><span class="s1">&#39;2023-10-11T21:48:00.000000&#39;</span>
+-<span class="w"> </span>Binary:<span class="w"> </span>heat-engine
+<span class="w">  </span>Engine<span class="w"> </span>ID:<span class="w"> </span>1011943b-9fea-4f53-b543-d841297245fd
+<span class="w">  </span>Host:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Hostname:<span class="w"> </span>heat-engine-6d47856868-p7pzz
+<span class="w">  </span>Status:<span class="w"> </span>up
+<span class="w">  </span>Topic:<span class="w"> </span>engine
+<span class="w">  </span>Updated<span class="w"> </span>At:<span class="w"> </span><span class="s1">&#39;2023-10-11T21:48:01.000000&#39;</span>
+</code></pre></div>
+<h3 id="verify-you-can-now-see-your-heat-stacks-again">Verify you can now see your Heat stacks again<a class="headerlink" href="#verify-you-can-now-see-your-heat-stacks-again" title="Permanent link">#</a></h3>
+<p>We can now test that user can create networks, subnets, ports, routers etc.</p>
+<div class="highlight"><pre><span></span><code>❯<span class="w"> </span>openstack<span class="w"> </span>stack<span class="w"> </span>list<span class="w"> </span>-f<span class="w"> </span>yaml
+-<span class="w"> </span>Creation<span class="w"> </span>Time:<span class="w"> </span><span class="s1">&#39;2023-10-11T22:03:20Z&#39;</span>
+<span class="w">  </span>ID:<span class="w"> </span>20f95925-7443-49cb-9561-a1ab736749ba
+<span class="w">  </span>Project:<span class="w"> </span>4eacd0d1cab04427bc315805c28e66c9
+<span class="w">  </span>Stack<span class="w"> </span>Name:<span class="w"> </span>test-networks
+<span class="w">  </span>Stack<span class="w"> </span>Status:<span class="w"> </span>CREATE_COMPLETE
+<span class="w">  </span>Updated<span class="w"> </span>Time:<span class="w"> </span>null
+</code></pre></div>
+
+  <hr>
+<div class="md-source-file">
+  <small>
+    
+      Last update:
+      2023-10-12
+    
+  </small>
+</div>
+
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+        </div>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "../..", "features": ["search.suggest", "search.highlight", "search.share"], "search": "../../assets/javascripts/workers/search.f886a092.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"default": "latest", "provider": "mike"}}</script>
+    
+    
+      <script src="../../assets/javascripts/bundle.aecac24b.min.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file
diff --git a/openstack/horizon_adoption/index.html b/openstack/horizon_adoption/index.html
index d6d8ae08e..374f19bdb 100644
--- a/openstack/horizon_adoption/index.html
+++ b/openstack/horizon_adoption/index.html
@@ -689,6 +689,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/ironic_adoption/index.html b/openstack/ironic_adoption/index.html
index c3d27065c..604003599 100644
--- a/openstack/ironic_adoption/index.html
+++ b/openstack/ironic_adoption/index.html
@@ -18,7 +18,7 @@
         <link rel="prev" href="../edpm_adoption/">
       
       
-        <link rel="next" href="../troubleshooting/">
+        <link rel="next" href="../heat_adoption/">
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
@@ -692,6 +692,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/keystone_adoption/index.html b/openstack/keystone_adoption/index.html
index 33a95600b..09626b425 100644
--- a/openstack/keystone_adoption/index.html
+++ b/openstack/keystone_adoption/index.html
@@ -692,6 +692,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/mariadb_copy/index.html b/openstack/mariadb_copy/index.html
index fc72fa6f1..7216cd3c0 100644
--- a/openstack/mariadb_copy/index.html
+++ b/openstack/mariadb_copy/index.html
@@ -696,6 +696,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/neutron_adoption/index.html b/openstack/neutron_adoption/index.html
index afb4a9c52..d9b249936 100644
--- a/openstack/neutron_adoption/index.html
+++ b/openstack/neutron_adoption/index.html
@@ -709,6 +709,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/node-selector/index.html b/openstack/node-selector/index.html
index 11ea872e5..7c9c530bc 100644
--- a/openstack/node-selector/index.html
+++ b/openstack/node-selector/index.html
@@ -615,6 +615,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/ovn_adoption/index.html b/openstack/ovn_adoption/index.html
index 41afbcc89..187ce1086 100644
--- a/openstack/ovn_adoption/index.html
+++ b/openstack/ovn_adoption/index.html
@@ -689,6 +689,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/placement_adoption/index.html b/openstack/placement_adoption/index.html
index 3c04fee1d..ab721883b 100644
--- a/openstack/placement_adoption/index.html
+++ b/openstack/placement_adoption/index.html
@@ -689,6 +689,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/planning/index.html b/openstack/planning/index.html
index 69c9df51d..f51afcaba 100644
--- a/openstack/planning/index.html
+++ b/openstack/planning/index.html
@@ -709,6 +709,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/pull_openstack_configuration/index.html b/openstack/pull_openstack_configuration/index.html
index 801045130..fff6a16f8 100644
--- a/openstack/pull_openstack_configuration/index.html
+++ b/openstack/pull_openstack_configuration/index.html
@@ -668,6 +668,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/stop_openstack_services/index.html b/openstack/stop_openstack_services/index.html
index 744a0fe02..461a1f7c2 100644
--- a/openstack/stop_openstack_services/index.html
+++ b/openstack/stop_openstack_services/index.html
@@ -682,6 +682,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
     <li class="md-nav__item">
       <a href="../troubleshooting/" class="md-nav__link">
         
diff --git a/openstack/troubleshooting/index.html b/openstack/troubleshooting/index.html
index d2c89742c..8f4e0429a 100644
--- a/openstack/troubleshooting/index.html
+++ b/openstack/troubleshooting/index.html
@@ -15,7 +15,7 @@
         <link rel="canonical" href="https://openstack-k8s-operators.github.io/data-plane-adoption/openstack/troubleshooting/">
       
       
-        <link rel="prev" href="../ironic_adoption/">
+        <link rel="prev" href="../heat_adoption/">
       
       
         <link rel="next" href="../../ceph/ceph_rbd/">
@@ -620,6 +620,26 @@
                 
   
   
+  
+    <li class="md-nav__item">
+      <a href="../heat_adoption/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Heat adoption
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
     
   
   
diff --git a/search/search_index.json b/search/search_index.json
index a1748e1f2..8d999bc9b 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Data Plane Adoption procedure","text":""},{"location":"#openstack-adoption","title":"OpenStack adoption","text":"<p>This is a procedure for adopting an OpenStack cloud.</p> <p>Perform the actions from the sub-documents in the following order:</p> <ul> <li> <p>Planning the new deployment</p> </li> <li> <p>Deploy podified backend services</p> </li> <li> <p>Pull Openstack configuration</p> </li> <li> <p>Stop OpenStack services</p> </li> <li> <p>Copy MariaDB data</p> </li> <li> <p>OVN adoption</p> </li> <li> <p>Keystone adoption</p> </li> <li> <p>Neutron adoption</p> </li> <li> <p>Ceph backend configuration (if applicable)</p> </li> <li> <p>Glance adoption</p> </li> <li> <p>Placement adoption</p> </li> <li> <p>Cinder adoption</p> </li> <li> <p>Horizon adoption</p> </li> <li> <p>Dataplane adoption</p> </li> <li> <p>Ironic adoption</p> </li> </ul> <p>If you face issues during adoption, check the Troubleshooting document for common problems and solutions.</p>"},{"location":"#post-openstack-ceph-adoption","title":"Post-OpenStack Ceph adoption","text":"<p>If the environment includes Ceph and some of its services are collocated on the Controller hosts (\"internal Ceph\"), then Ceph services need to be moved out of Controller hosts as the last step of the OpenStack adoption. Follow this documentation:</p> <ul> <li>Ceph RBD migration</li> <li>Ceph RGW migration</li> </ul>"},{"location":"#contributing","title":"Contributing","text":"<p>For information about contributing to the docs and how to run tests, see:</p> <ul> <li> <p>Contributing to documentation -   how to build docs locally, docs patterns and tips.</p> </li> <li> <p>Development environment -   how set up a local development environment where Adoption can be   executed (either manually or via the test suite).</p> </li> <li> <p>Tests -   information about the test suite and how to run it.</p> </li> </ul>"},{"location":"ceph/ceph_rbd/","title":"Data Plane adoption - Ceph RBD Migration","text":"<p>In this scenario, assuming Ceph is already &gt;= 5, either for HCI or dedicated Storage nodes, the daemons living in the OpenStack control plane should be moved/migrated into the existing external RHEL nodes (typically the compute nodes for an HCI environment or dedicated storage nodes in all the remaining use cases).</p>"},{"location":"ceph/ceph_rbd/#requirements","title":"Requirements","text":"<ul> <li>Ceph is &gt;= 5 and managed by cephadm/orchestrator.</li> <li>Ceph NFS (ganesha) migrated from a TripleO based deployment to cephadm.</li> <li>Both the Ceph public and cluster networks are propagated, via TripleO, to the target nodes.</li> <li>Ceph Mons need to keep their IPs (to avoid cold migration).</li> </ul>"},{"location":"ceph/ceph_rbd/#scenario-1-migrate-mon-and-mgr-from-controller-nodes","title":"SCENARIO 1: Migrate mon and mgr from controller nodes","text":"<p>The goal of the first POC is to prove we are able to successfully drain a controller node, in terms of ceph daemons, and move them to a different node. The initial target of the POC is RBD only, which means we\u2019re going to move only mon and mgr daemons. For the purposes of this POC, we'll deploy a ceph cluster with only mon, mgrs, and osds to simulate the environment a customer will be in before starting the migration. The goal of the first POC is to ensure that: - We can keep the mon IP addresses moving them to the CephStorage nodes. - We can drain the existing controller nodes and shut them down. - We can deploy additional monitors to the existing nodes, promoting them as   _admin nodes that can be used by administrators to manage the ceph cluster   and perform day2 operations against it. - We can keep the cluster operational during the migration.</p>"},{"location":"ceph/ceph_rbd/#prerequisites","title":"Prerequisites","text":"<p>The Storage Nodes should be configured to have both storage and storage_mgmt network to make sure we can use both Ceph public and cluster networks.</p> <p>This step is the only one where the interaction with TripleO is required. From 17+ we don\u2019t have to run any stack update, however, we have commands that should be performed to run os-net-config on the bare-metal node and configure additional networks.</p> <p>Make sure the network is defined in metalsmith.yaml for the CephStorageNodes:</p> <pre><code>- name: CephStorage\n  count: 2\n  instances:\n    - hostname: oc0-ceph-0\n      name: oc0-ceph-0\n    - hostname: oc0-ceph-1\n      name: oc0-ceph-1\n  defaults:\n    networks:\n      - network: ctlplane\n        vif: true\n      - network: storage_cloud_0\n          subnet: storage_cloud_0_subnet\n      - network: storage_mgmt_cloud_0\n          subnet: storage_mgmt_cloud_0_subnet\n    network_config:\n      template: templates/single_nic_vlans/single_nic_vlans_storage.j2\n</code></pre> <p>Then run:</p> <pre><code>openstack overcloud node provision \\\n  -o overcloud-baremetal-deployed-0.yaml --stack overcloud-0 \\\n  --network-config -y --concurrency 2 /home/stack/metalsmith-0.yam\n</code></pre> <p>Verify that the storage network is running on the node:</p> <pre><code>(undercloud) [CentOS-9 - stack@undercloud ~]$ ssh heat-admin@192.168.24.14 ip -o -4 a\nWarning: Permanently added '192.168.24.14' (ED25519) to the list of known hosts.\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n</code></pre>"},{"location":"ceph/ceph_rbd/#migrate-mons-and-mgrs-on-the-two-existing-cephstorage-nodes","title":"Migrate mon(s) and mgr(s) on the two existing CephStorage nodes","text":"<p>Create a ceph spec based on the default roles with the mon/mgr on the controller nodes.</p> <pre><code>openstack overcloud ceph spec -o ceph_spec.yaml -y  \\\n   --stack overcloud-0     overcloud-baremetal-deployed-0.yaml\n</code></pre> <p>Deploy the Ceph cluster</p> <pre><code> openstack overcloud ceph deploy overcloud-baremetal-deployed-0.yaml \\\n    --stack overcloud-0 -o deployed_ceph.yaml \\\n    --network-data ~/oc0-network-data.yaml \\\n    --ceph-spec ~/ceph_spec.yaml\n</code></pre> <p>Note:</p> <p>The ceph_spec.yaml, which is the OSP-generated description of the ceph cluster, will be used, later in the process, as the basic template required by cephadm to update the status/info of the daemons.</p> <p>Check the status of the cluster:</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_OK\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-1,oc0-controller-2 (age 19m)\n    mgr: oc0-controller-0.xzgtvo(active, since 32m), standbys: oc0-controller-1.mtxohd, oc0-controller-2.ahrgsk\n    osd: 8 osds: 8 up (since 12m), 8 in (since 18m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host ls\nHOST              ADDR           LABELS          STATUS\noc0-ceph-0        192.168.24.14  osd\noc0-ceph-1        192.168.24.7   osd\noc0-controller-0  192.168.24.15  _admin mgr mon\noc0-controller-1  192.168.24.23  _admin mgr mon\noc0-controller-2  192.168.24.13  _admin mgr mon\n</code></pre> <p>The goal of the next section is to migrate the oc0-controller-{1,2} daemons into oc0-ceph-{0,1} as the very basic scenario that demonstrates we can actually make this kind of migration using cephadm.</p>"},{"location":"ceph/ceph_rbd/#migrate-oc0-controller-1-into-oc0-ceph-0","title":"Migrate oc0-controller-1 into oc0-ceph-0","text":"<p>ssh into controller-0, then</p> <p><code>cephadm shell -v /home/ceph-admin/specs:/specs</code></p> <p>ssh into ceph-0, then</p> <p><code>sudo \u201cwatch podman ps\u201d  # watch the new mon/mgr being deployed here</code></p> <p>(optional) if mgr is active in the source node, then:</p> <pre><code>ceph mgr fail &lt;mgr instance&gt;\n</code></pre> <p>From the cephadm shell, remove the labels on oc0-controller-1</p> <pre><code>    for label in mon mgr _admin; do\n           ceph orch host rm label oc0-controller-1 $label;\n    done\n</code></pre> <p>Add the missing labels to oc0-ceph-0</p> <pre><code>[ceph: root@oc0-controller-0 /]#\n&gt; for label in mon mgr _admin; do ceph orch host label add oc0-ceph-0 $label; done\nAdded label mon to host oc0-ceph-0\nAdded label mgr to host oc0-ceph-0\nAdded label _admin to host oc0-ceph-0\n</code></pre> <p>Drain and force-remove the oc0-controller-1 node</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host drain oc0-controller-1\nScheduled to remove the following daemons from host 'oc0-controller-1'\ntype                 id\n-------------------- ---------------\nmon                  oc0-controller-1\nmgr                  oc0-controller-1.mtxohd\ncrash                oc0-controller-1\n</code></pre> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host rm oc0-controller-1 --force\nRemoved  host 'oc0-controller-1'\n\n[ceph: root@oc0-controller-0 /]# ceph orch host ls\nHOST              ADDR           LABELS          STATUS\noc0-ceph-0        192.168.24.14  osd\noc0-ceph-1        192.168.24.7   osd\noc0-controller-0  192.168.24.15  mgr mon _admin\noc0-controller-2  192.168.24.13  _admin mgr mon\n</code></pre> <p>If you have only 3 mon nodes, and the drain of the node doesn\u2019t work as expected (the containers are still there), then SSH to controller-1 and force-purge the containers in the node:</p> <pre><code>[root@oc0-controller-1 ~]# sudo podman ps\nCONTAINER ID  IMAGE                                                                                        COMMAND               CREATED         STATUS             PORTS       NAMES\n5c1ad36472bc  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mon.oc0-contro...  35 minutes ago  Up 35 minutes ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mon-oc0-controller-1\n3b14cc7bf4dd  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mgr.oc0-contro...  35 minutes ago  Up 35 minutes ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mgr-oc0-controller-1-mtxohd\n\n[root@oc0-controller-1 ~]# cephadm rm-cluster --fsid f6ec3ebe-26f7-56c8-985d-eb974e8e08e3 --force\n\n[root@oc0-controller-1 ~]# sudo podman ps\nCONTAINER ID  IMAGE       COMMAND     CREATED     STATUS      PORTS       NAMES\n</code></pre> <p>Note: cephadm rm-cluster on a node that is not part of the cluster anymore has the effect of removing all the containers and doing some cleanup on the filesystem.</p> <p>Before shutting the oc0-controller-1 down, move the IP address (on the same network) to the oc0-ceph-0 node:</p> <pre><code>mon_host = [v2:172.16.11.54:3300/0,v1:172.16.11.54:6789/0] [v2:172.16.11.121:3300/0,v1:172.16.11.121:6789/0] [v2:172.16.11.205:3300/0,v1:172.16.11.205:6789/0]\n\n[root@oc0-controller-1 ~]# ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-ex    inet 192.168.24.23/24 brd 192.168.24.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n6: vlan100    inet 192.168.100.96/24 brd 192.168.100.255 scope global vlan100\\       valid_lft forever preferred_lft forever\n7: vlan12    inet 172.16.12.154/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n8: vlan11    inet 172.16.11.121/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n9: vlan13    inet 172.16.13.178/24 brd 172.16.13.255 scope global vlan13\\       valid_lft forever preferred_lft forever\n10: vlan70    inet 172.17.0.23/20 brd 172.17.15.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n11: vlan1    inet 192.168.24.23/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n12: vlan14    inet 172.16.14.223/24 brd 172.16.14.255 scope global vlan14\\       valid_lft forever preferred_lft forever\n</code></pre> <p>On the oc0-ceph-0:</p> <pre><code>[heat-admin@oc0-ceph-0 ~]$ ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n[heat-admin@oc0-ceph-0 ~]$ sudo ip a add 172.16.11.121 dev vlan11\n[heat-admin@oc0-ceph-0 ~]$ ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.121/32 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n</code></pre> <p>Poweroff oc0-controller-1.</p> <p>Add the new mon on oc0-ceph-0 using the old IP address:</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch daemon add mon oc0-ceph-0:172.16.11.121\nDeployed mon.oc0-ceph-0 on host 'oc0-ceph-0'\n</code></pre> <p>Check the new container in the oc0-ceph-0 node:</p> <pre><code>b581dc8bbb78  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mon.oc0-ceph-0...  24 seconds ago  Up 24 seconds ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mon-oc0-ceph-0\n</code></pre> <p>On the cephadm shell, backup the existing ceph_spec.yaml, edit the spec removing any oc0-controller-1 entry, and replacing it with oc0-ceph-0:</p> <pre><code>cp ceph_spec.yaml ceph_spec.yaml.bkp # backup the ceph_spec.yaml file\n\n[ceph: root@oc0-controller-0 specs]# diff -u ceph_spec.yaml.bkp ceph_spec.yaml\n\n--- ceph_spec.yaml.bkp  2022-07-29 15:41:34.516329643 +0000\n+++ ceph_spec.yaml      2022-07-29 15:28:26.455329643 +0000\n@@ -7,14 +7,6 @@\n - mgr\n service_type: host\n ---\n-addr: 192.168.24.12\n-hostname: oc0-controller-1\n-labels:\n-- _admin\n-- mon\n-- mgr\n-service_type: host\n----\n addr: 192.168.24.19\n hostname: oc0-controller-2\n labels:\n@@ -38,7 +30,7 @@\n placement:\n   hosts:\n   - oc0-controller-0\n-  - oc0-controller-1\n+  - oc0-ceph-0\n   - oc0-controller-2\n service_id: mon\n service_name: mon\n@@ -47,8 +39,8 @@\n placement:\n   hosts:\n   - oc0-controller-0\n-  - oc0-controller-1\n   - oc0-controller-2\n+  - oc0-ceph-0\n service_id: mgr\n service_name: mgr\n service_type: mgr\n</code></pre> <p>Apply the resulting spec:</p> <pre><code>ceph orch apply -i ceph_spec.yaml \n\n The result of 12 is having a new mgr deployed on the oc0-ceph-0 node, and the spec reconciled within cephadm\n\n[ceph: root@oc0-controller-0 specs]# ceph orch ls\nNAME                     PORTS  RUNNING  REFRESHED  AGE  PLACEMENT\ncrash                               4/4  5m ago     61m  *\nmgr                                 3/3  5m ago     69s  oc0-controller-0;oc0-ceph-0;oc0-controller-2\nmon                                 3/3  5m ago     70s  oc0-controller-0;oc0-ceph-0;oc0-controller-2\nosd.default_drive_group               8  2m ago     69s  oc0-ceph-0;oc0-ceph-1\n\n[ceph: root@oc0-controller-0 specs]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_WARN\n            1 stray host(s) with 1 daemon(s) not managed by cephadm\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-2,oc0-ceph-0 (age 5m)\n    mgr: oc0-controller-0.xzgtvo(active, since 62m), standbys: oc0-controller-2.ahrgsk, oc0-ceph-0.hccsbb\n    osd: 8 osds: 8 up (since 42m), 8 in (since 49m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <p>Fix the warning by refreshing the mgr:</p> <pre><code>ceph mgr fail oc0-controller-0.xzgtvo\n</code></pre> <p>And at this point the cluster is clean:</p> <pre><code>[ceph: root@oc0-controller-0 specs]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_OK\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-2,oc0-ceph-0 (age 7m)\n    mgr: oc0-controller-2.ahrgsk(active, since 25s), standbys: oc0-controller-0.xzgtvo, oc0-ceph-0.hccsbb\n    osd: 8 osds: 8 up (since 44m), 8 in (since 50m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <p>oc0-controller-1 has been removed and powered off without leaving traces on the ceph cluster.</p> <p>The same approach and the same steps can be applied to migrate oc0-controller-2 to oc0-ceph-1.</p>"},{"location":"ceph/ceph_rbd/#screen-recording","title":"Screen Recording:","text":"<ul> <li>Externalize a TripleO deployed Ceph cluster</li> </ul>"},{"location":"ceph/ceph_rbd/#whats-next","title":"What\u2019s next","text":""},{"location":"ceph/ceph_rbd/#useful-resources","title":"Useful resources","text":"<ul> <li>cephadm - deploy additional mon(s)</li> </ul>"},{"location":"ceph/ceph_rgw/","title":"Data Plane adoption - Ceph RGW Migration","text":"<p>In this scenario, assuming Ceph is already &gt;= 5, either for HCI or dedicated Storage nodes, the RGW daemons living in the OpenStack Controller nodes will be migrated into the existing external RHEL nodes (typically the Compute nodes for an HCI environment or CephStorage nodes in the remaining use cases).</p>"},{"location":"ceph/ceph_rgw/#requirements","title":"Requirements","text":"<ul> <li>Ceph is &gt;= 5 and managed by cephadm/orchestrator</li> <li>An undercloud is still available: nodes and networks are managed by TripleO</li> </ul>"},{"location":"ceph/ceph_rgw/#ceph-daemon-cardinality","title":"Ceph Daemon Cardinality","text":"<p>Ceph 5+ applies strict constraints in the way daemons can be colocated within the same node. The resulting topology depends on the available hardware, as well as the amount of Ceph services present in the Controller nodes which are going to be retired. The following document describes the procedure required to migrate the RGW component (and keep an HA model using the Ceph Ingress daemon in a common TripleO scenario where Controller nodes represent the spec placement where the service is deployed. As a general rule, the number of services that can be migrated depends on the number of available nodes in the cluster. The following diagrams cover the distribution of the Ceph daemons on the CephStorage nodes where at least three nodes are required in a scenario that sees only RGW and RBD (no dashboard):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress <p>With dashboard, and without Manila at least four nodes are required (dashboard has no failover):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash dashboard/grafana osd rgw/ingress (free) <p>With dashboard and Manila 5 nodes minimum are required (and dashboard has no failover):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash mds/ganesha/ingress osd rgw/ingress mds/ganesha/ingress osd mds/ganesha/ingress dashboard/grafana"},{"location":"ceph/ceph_rgw/#current-status","title":"Current Status","text":"<pre><code>(undercloud) [stack@undercloud-0 ~]$ metalsmith list\n\n\n    +------------------------+    +----------------+\n    | IP Addresses           |    |  Hostname      |\n    +------------------------+    +----------------+\n    | ctlplane=192.168.24.25 |    | cephstorage-0  |\n    | ctlplane=192.168.24.10 |    | cephstorage-1  |\n    | ctlplane=192.168.24.32 |    | cephstorage-2  |\n    | ctlplane=192.168.24.28 |    | compute-0      |\n    | ctlplane=192.168.24.26 |    | compute-1      |\n    | ctlplane=192.168.24.43 |    | controller-0   |\n    | ctlplane=192.168.24.7  |    | controller-1   |\n    | ctlplane=192.168.24.41 |    | controller-2   |\n    +------------------------+    +----------------+\n</code></pre> <p>SSH into <code>controller-0</code> and check the <code>pacemaker</code> status: this will help identify the relevant information that we need to know before starting the RGW migration.</p> <pre><code>Full List of Resources:\n  * ip-192.168.24.46    (ocf:heartbeat:IPaddr2):        Started controller-0\n  * ip-10.0.0.103       (ocf:heartbeat:IPaddr2):        Started controller-1\n  * ip-172.17.1.129     (ocf:heartbeat:IPaddr2):        Started controller-2\n  * ip-172.17.3.68      (ocf:heartbeat:IPaddr2):        Started controller-0\n  * ip-172.17.4.37      (ocf:heartbeat:IPaddr2):        Started controller-1\n  * Container bundle set: haproxy-bundle\n\n[undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:\n    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-2\n    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-0\n    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-1\n</code></pre> <p>Use the <code>ip</code> command to identify the ranges of the storage networks.</p> <pre><code>[heat-admin@controller-0 ~]$ ip -o -4 a\n\n1: lo   inet 127.0.0.1/8 scope host lo\\     valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.45/24 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.46/32 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n7: br-ex    inet 10.0.0.122/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n8: vlan70   inet 172.17.5.22/24 brd 172.17.5.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n8: vlan70   inet 172.17.5.94/32 brd 172.17.5.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n9: vlan50   inet 172.17.2.140/24 brd 172.17.2.255 scope global vlan50\\      valid_lft forever preferred_lft forever\n10: vlan30  inet 172.17.3.73/24 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n10: vlan30  inet 172.17.3.68/32 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n11: vlan20  inet 172.17.1.88/24 brd 172.17.1.255 scope global vlan20\\       valid_lft forever preferred_lft forever\n12: vlan40  inet 172.17.4.24/24 brd 172.17.4.255 scope global vlan40\\       valid_lft forever preferred_lft forever\n</code></pre> <p>In this example:</p> <ul> <li>vlan30 represents the Storage Network, where the new RGW instances should be   started on the CephStorage nodes</li> <li>br-ex represents the External Network, which is where in the current   environment, haproxy has the frontend VIP assigned</li> </ul>"},{"location":"ceph/ceph_rgw/#prerequisite-check-the-frontend-network-controller-nodes","title":"Prerequisite: check the frontend network (Controller nodes)","text":"<p>Identify the network that we previously had in haproxy and propagate it (via TripleO) to the CephStorage nodes. This network is used to reserve a new VIP that will be owned by Ceph and used as the entry point for the RGW service.</p> <p>ssh into <code>controller-0</code> and check the current HaProxy configuration until we find <code>ceph_rgw</code> section:</p> <pre><code>$ less /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg\n\n...\n...\nlisten ceph_rgw\n  bind 10.0.0.103:8080 transparent\n  bind 172.17.3.68:8080 transparent\n  mode http\n  balance leastconn\n  http-request set-header X-Forwarded-Proto https if { ssl_fc }\n  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }\n  http-request set-header X-Forwarded-Port %[dst_port]\n  option httpchk GET /swift/healthcheck\n  option httplog\n  option forwardfor\n  server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2\n  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2\n  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2\n</code></pre> <p>Double check the network used as HaProxy frontend:</p> <pre><code>[controller-0]$ ip -o -4 a\n\n...\n7: br-ex    inet 10.0.0.106/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n...\n</code></pre> <p>As described in the previous section, the check on controller-0 shows that we are exposing the services using the external network, which is not present in the CephStorage nodes, and we need to propagate it via TripleO.</p>"},{"location":"ceph/ceph_rgw/#propagate-the-haproxy-frontend-network-to-cephstorage-nodes","title":"Propagate the <code>HaProxy</code> frontend network to <code>CephStorage</code> nodes","text":"<p>Change the nic template used to define the ceph-storage network interfaces and add the new config section.</p> <pre><code>---\nnetwork_config:\n- type: interface\n  name: nic1\n  use_dhcp: false\n  dns_servers: {{ ctlplane_dns_nameservers }}\n  addresses:\n  - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}\n  routes: {{ ctlplane_host_routes }}\n- type: vlan\n  vlan_id: {{ storage_mgmt_vlan_id }}\n  device: nic1\n  addresses:\n  - ip_netmask: {{ storage_mgmt_ip }}/{{ storage_mgmt_cidr }}\n  routes: {{ storage_mgmt_host_routes }}\n- type: interface\n  name: nic2\n  use_dhcp: false\n  defroute: false\n- type: vlan\n  vlan_id: {{ storage_vlan_id }}\n  device: nic2\n  addresses:\n  - ip_netmask: {{ storage_ip }}/{{ storage_cidr }}\n  routes: {{ storage_host_routes }}\n- type: ovs_bridge\n  name: {{ neutron_physical_bridge_name }}\n  dns_servers: {{ ctlplane_dns_nameservers }}\n  domain: {{ dns_search_domains }}\n  use_dhcp: false\n  addresses:\n  - ip_netmask: {{ external_ip }}/{{ external_cidr }}\n  routes: {{ external_host_routes }}\n  members:\n  - type: interface\n    name: nic3\n    primary: true\n</code></pre> <p>In addition, add the External Network to the <code>baremetal.yaml</code> file used by metalsmith and run the <code>overcloud node provision</code> command passing the <code>--network-config</code> option:</p> <pre><code>- name: CephStorage\n  count: 3\n  hostname_format: cephstorage-%index%\n  instances:\n  - hostname: cephstorage-0\n  name: ceph-0\n  - hostname: cephstorage-1\n  name: ceph-1\n  - hostname: cephstorage-2\n  name: ceph-2\n  defaults:\n  profile: ceph-storage\n  network_config:\n      template: /home/stack/composable_roles/network/nic-configs/ceph-storage.j2\n  networks:\n  - network: ctlplane\n      vif: true\n  - network: storage\n  - network: storage_mgmt\n  - network: external\n</code></pre> <pre><code>(undercloud) [stack@undercloud-0]$\n\nopenstack overcloud node provision\n   -o overcloud-baremetal-deployed-0.yaml\n   --stack overcloud\n   --network-config -y\n  $PWD/network/baremetal_deployment.yaml\n</code></pre> <p>Check the new network on the <code>CephStorage</code> nodes:</p> <pre><code>[root@cephstorage-0 ~]# ip -o -4 a\n\n1: lo   inet 127.0.0.1/8 scope host lo\\     valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.54/24 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n11: vlan40  inet 172.17.4.43/24 brd 172.17.4.255 scope global vlan40\\       valid_lft forever preferred_lft forever\n12: vlan30  inet 172.17.3.23/24 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n14: br-ex   inet 10.0.0.133/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n</code></pre> <p>And now it\u2019s time to start migrating the RGW backends and build the ingress on top of them.</p>"},{"location":"ceph/ceph_rgw/#migrate-the-rgw-backends","title":"Migrate the RGW backends","text":"<p>To match the cardinality diagram we use cephadm labels to refer to a group of nodes where a given daemon type should be deployed.</p> <p>Add the RGW label to the cephstorage nodes:</p> <pre><code>for i in 0 1 2; {\n    ceph orch host label add cephstorage-$i rgw;\n}\n</code></pre> <pre><code>[ceph: root@controller-0 /]#\n\nfor i in 0 1 2; {\n    ceph orch host label add cephstorage-$i rgw;\n}\n\nAdded label rgw to host cephstorage-0\nAdded label rgw to host cephstorage-1\nAdded label rgw to host cephstorage-2\n\n[ceph: root@controller-0 /]# ceph orch host ls\n\nHOST        ADDR        LABELS          STATUS\ncephstorage-0  192.168.24.54  osd rgw\ncephstorage-1  192.168.24.44  osd rgw\ncephstorage-2  192.168.24.30  osd rgw\ncontroller-0   192.168.24.45  _admin mon mgr\ncontroller-1   192.168.24.11  _admin mon mgr\ncontroller-2   192.168.24.38  _admin mon mgr\n\n6 hosts in cluster\n</code></pre> <p>During the overcloud deployment, RGW is applied at step2 (external_deployment_steps), and a cephadm compatible spec is generated in <code>/home/ceph-admin/specs/rgw</code> from the ceph_mkspec ansible module. Find and patch the RGW spec, specifying the right placement using the labels approach, and change the rgw backend port to 8090 to avoid conflicts with the Ceph Ingress Daemon (*)</p> <pre><code>[root@controller-0 heat-admin]# cat rgw\n\nnetworks:\n- 172.17.3.0/24\nplacement:\n  hosts:\n  - controller-0\n  - controller-1\n  - controller-2\nservice_id: rgw\nservice_name: rgw.rgw\nservice_type: rgw\nspec:\n  rgw_frontend_port: 8080\n  rgw_realm: default\n  rgw_zone: default\n</code></pre> <p>Patch the spec replacing controller nodes with the label key</p> <pre><code>---\nnetworks:\n- 172.17.3.0/24\nplacement:\n  label: rgw\nservice_id: rgw\nservice_name: rgw.rgw\nservice_type: rgw\nspec:\n  rgw_frontend_port: 8090\n  rgw_realm: default\n  rgw_zone: default\n</code></pre> <p>(*) cephadm_check_port</p> <p>Apply the new RGW spec using the orchestrator CLI:</p> <pre><code>$ cephadm shell -m /home/ceph-admin/specs/rgw\n$ cephadm shell -- ceph orch apply -i /mnt/rgw\n</code></pre> <p>Which triggers the redeploy:</p> <pre><code>...\nosd.9                       cephstorage-2\nrgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090   starting\nrgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090   starting\nrgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090   starting\nrgw.rgw.controller-1.eyvrzw   controller-1   172.17.3.146:8080  running (5h)\nrgw.rgw.controller-2.navbxa   controller-2   172.17.3.66:8080   running (5h)\n\n...\nosd.9                       cephstorage-2\nrgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090  running (19s)\nrgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090  running (16s)\nrgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090  running (13s)\n</code></pre> <p>At this point, we need to make sure that the new RGW backends are reachable on the new ports, but we\u2019re going to enable an IngressDaemon on port 8080 later in the process. For this reason, ssh on each RGW node (the CephStorage nodes) and add the iptables rule to allow connections to both 8080 and 8090 ports in the CephStorage nodes.</p> <pre><code>iptables -I INPUT -p tcp -m tcp --dport 8080 -m conntrack --ctstate NEW -m comment --comment \"ceph rgw ingress\" -j ACCEPT\n\niptables -I INPUT -p tcp -m tcp --dport 8090 -m conntrack --ctstate NEW -m comment --comment \"ceph rgw backends\" -j ACCEPT\n\nfor port in 8080 8090; { \n    for i in 25 10 32; {\n       ssh heat-admin@192.168.24.$i sudo iptables -I INPUT \\\n       -p tcp -m tcp --dport $port -m conntrack --ctstate NEW \\\n       -j ACCEPT;\n   }\n}\n</code></pre> <p>From a Controller node (e.g. controller-0) try to reach (curl) the rgw backends:</p> <pre><code>for i in 26 23 81; do {\n    echo \"----\"\n    curl 172.17.3.$i:8090;\n    echo \"----\"\n    echo\ndone\n</code></pre> <p>And you should observe the following:</p> <pre><code>----\nQuery 172.17.3.23\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n\n----\nQuery 172.17.3.26\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n\n----\nQuery 172.17.3.81\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n</code></pre>"},{"location":"ceph/ceph_rgw/#note","title":"NOTE","text":"<p>In case RGW backends are migrated in the CephStorage nodes, there\u2019s no \u201cinternalAPI\u201d network(this is not true in the case of HCI). Reconfig the RGW keystone endpoint, pointing to the external Network that has been propagated (see the previous section)</p> <pre><code>[ceph: root@controller-0 /]# ceph config dump | grep keystone\nglobal   basic rgw_keystone_url  http://172.16.1.111:5000\n\n[ceph: root@controller-0 /]# ceph config set global rgw_keystone_url http://10.0.0.103:5000\n</code></pre>"},{"location":"ceph/ceph_rgw/#deploy-a-ceph-ingressdaemon","title":"Deploy a Ceph IngressDaemon","text":"<p><code>HaProxy</code> is managed by TripleO via <code>Pacemaker</code>: the three running instances at this point will point to the old RGW backends, resulting in a wrong, not working configuration. Since we\u2019re going to deploy the Ceph Ingress Daemon, the first thing to do is remove the existing <code>ceph_rgw</code> config, clean up the config created by TripleO and restart the service to make sure other services are not affected by this change.</p> <p>ssh  on each Controller node and remove the following is the section from <code>/var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg</code>:</p> <pre><code>listen ceph_rgw\n  bind 10.0.0.103:8080 transparent\n  mode http\n  balance leastconn\n  http-request set-header X-Forwarded-Proto https if { ssl_fc }\n  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }\n  http-request set-header X-Forwarded-Port %[dst_port]\n  option httpchk GET /swift/healthcheck\n  option httplog\n  option forwardfor\n   server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2\n  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2\n  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2\n</code></pre> <p>Restart <code>haproxy-bundle</code> and make sure it\u2019s started:</p> <pre><code>[root@controller-0 ~]# sudo pcs resource restart haproxy-bundle\nhaproxy-bundle successfully restarted\n\n\n[root@controller-0 ~]# sudo pcs status | grep haproxy\n\n  * Container bundle set: haproxy-bundle [undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:\n    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-0\n    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-1\n    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-2\n</code></pre> <p>Double check no process is bound to 8080 anymore\u201d</p> <pre><code>[root@controller-0 ~]# ss -antop | grep 8080\n[root@controller-0 ~]#\n</code></pre> <p>And the swift CLI should fail at this point:</p> <pre><code>(overcloud) [root@cephstorage-0 ~]# swift list\n\nHTTPConnectionPool(host='10.0.0.103', port=8080): Max retries exceeded with url: /swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json (Caused by NewConnectionError('&lt;urllib3.connection.HTTPConnection object at 0x7fc41beb0430&gt;: Failed to establish a new connection: [Errno 111] Connection refused'))\n</code></pre> <p>Now we can start deploying the Ceph IngressDaemon on the CephStorage nodes.</p> <p>Set the required images for both HaProxy and Keepalived</p> <pre><code>[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_haproxy quay.io/ceph/haproxy:2.3\n\n[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_keepalived quay.io/ceph/keepalived:2.1.5\n</code></pre> <p>Prepare the ingress spec and mount it to cephadm:</p> <pre><code>$ sudo vim /home/ceph-admin/specs/rgw_ingress\n</code></pre> <p>and paste the following content:</p> <pre><code>---\nservice_type: ingress\nservice_id: rgw.rgw\nplacement:\n  label: rgw\nspec:\n  backend_service: rgw.rgw\n  virtual_ip: 10.0.0.89/24\n  frontend_port: 8080\n  monitor_port: 8898\n  virtual_interface_networks:\n    - 10.0.0.0/24\n</code></pre> <p>Mount the generated spec and apply it using the orchestrator CLI:</p> <pre><code>$ cephadm shell -m /home/ceph-admin/specs/rgw_ingress\n$ cephadm shell -- ceph orch apply -i /mnt/rgw_ingress\n</code></pre> <p>Wait until the ingress is deployed and query the resulting endpoint:</p> <pre><code>[ceph: root@controller-0 /]# ceph orch ls\n\nNAME                    PORTS               RUNNING  REFRESHED  AGE  PLACEMENT\ncrash                                           6/6  6m ago     3d   *\ningress.rgw.rgw         10.0.0.89:8080,8898     6/6  37s ago    60s  label:rgw\nmds.mds                   3/3  6m ago   3d   controller-0;controller-1;controller-2\nmgr                       3/3  6m ago   3d   controller-0;controller-1;controller-2\nmon                       3/3  6m ago   3d   controller-0;controller-1;controller-2\nosd.default_drive_group   15  37s ago   3d   cephstorage-0;cephstorage-1;cephstorage-2\nrgw.rgw   ?:8090          3/3  37s ago  4m   label:rgw\n</code></pre> <pre><code>[ceph: root@controller-0 /]# curl  10.0.0.89:8080\n\n---\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;[ceph: root@controller-0 /]#\n\u2014\n</code></pre> <p>The result above shows that we\u2019re able to reach the backend from the IngressDaemon, which means we\u2019re almost ready to interact with it using the swift CLI.</p>"},{"location":"ceph/ceph_rgw/#update-the-object-store-endpoints","title":"Update the object-store endpoints","text":"<p>The endpoints still point to the old VIP owned by pacemaker, but given it\u2019s still used by other services and we reserved a new VIP on the same network, before any other action we should update the object-store endpoint.</p> <p>List the current endpoints:</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object\n\n| 1326241fb6b6494282a86768311f48d1 | regionOne | swift      | object-store   | True | internal  | http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |\n| 8a34817a9d3443e2af55e108d63bb02b | regionOne | swift      | object-store   | True | public    | http://10.0.0.103:8080/swift/v1/AUTH_%(project_id)s  |\n| fa72f8b8b24e448a8d4d1caaeaa7ac58 | regionOne | swift      | object-store   | True | admin     | http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |\n</code></pre> <p>Update the endpoints pointing to the Ingress VIP:</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ openstack endpoint set --url \"http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s\" 95596a2d92c74c15b83325a11a4f07a3\n\n(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object-store\n| 6c7244cc8928448d88ebfad864fdd5ca | regionOne | swift      | object-store   | True | internal  | http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |\n| 95596a2d92c74c15b83325a11a4f07a3 | regionOne | swift      | object-store   | True | public    | http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s   |\n| e6d0599c5bf24a0fb1ddf6ecac00de2d | regionOne | swift      | object-store   | True | admin     | http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |\n</code></pre> <p>And repeat the same action for both internal and admin. Test the migrated service.</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ swift list --debug\n\nDEBUG:swiftclient:Versionless auth_url - using http://10.0.0.115:5000/v3 as endpoint\nDEBUG:keystoneclient.auth.identity.v3.base:Making authentication request to http://10.0.0.115:5000/v3/auth/tokens\nDEBUG:urllib3.connectionpool:Starting new HTTP connection (1): 10.0.0.115:5000\nDEBUG:urllib3.connectionpool:http://10.0.0.115:5000 \"POST /v3/auth/tokens HTTP/1.1\" 201 7795\nDEBUG:keystoneclient.auth.identity.v3.base:{\"token\": {\"methods\": [\"password\"], \"user\": {\"domain\": {\"id\": \"default\", \"name\": \"Default\"}, \"id\": \"6f87c7ffdddf463bbc633980cfd02bb3\", \"name\": \"admin\", \"password_expires_at\": null}, \n\n\n...\n...\n...\n\nDEBUG:swiftclient:REQ: curl -i http://10.0.0.89:8080/swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json -X GET -H \"X-Auth-Token: gAAAAABj7KHdjZ95syP4c8v5a2zfXckPwxFQZYg0pgWR42JnUs83CcKhYGY6PFNF5Cg5g2WuiYwMIXHm8xftyWf08zwTycJLLMeEwoxLkcByXPZr7kT92ApT-36wTfpi-zbYXd1tI5R00xtAzDjO3RH1kmeLXDgIQEVp0jMRAxoVH4zb-DVHUos\" -H \"Accept-Encoding: gzip\"\nDEBUG:swiftclient:RESP STATUS: 200 OK\nDEBUG:swiftclient:RESP HEADERS: {'content-length': '2', 'x-timestamp': '1676452317.72866', 'x-account-container-count': '0', 'x-account-object-count': '0', 'x-account-bytes-used': '0', 'x-account-bytes-used-actual': '0', 'x-account-storage-policy-default-placement-container-count': '0', 'x-account-storage-policy-default-placement-object-count': '0', 'x-account-storage-policy-default-placement-bytes-used': '0', 'x-account-storage-policy-default-placement-bytes-used-actual': '0', 'x-trans-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'x-openstack-request-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'accept-ranges': 'bytes', 'content-type': 'application/json; charset=utf-8', 'date': 'Wed, 15 Feb 2023 09:11:57 GMT'}\nDEBUG:swiftclient:RESP BODY: b'[]'\n</code></pre> <p>Run tempest tests against object-storage:</p> <pre><code>(overcloud) [stack@undercloud-0 tempest-dir]$  tempest run --regex tempest.api.object_storage\n...\n...\n...\n======\nTotals\n======\nRan: 141 tests in 606.5579 sec.\n - Passed: 128\n - Skipped: 13\n - Expected Fail: 0\n - Unexpected Success: 0\n - Failed: 0\nSum of execute time for each test: 657.5183 sec.\n\n==============\nWorker Balance\n==============\n - Worker 0 (1 tests) =&gt; 0:10:03.400561\n - Worker 1 (2 tests) =&gt; 0:00:24.531916\n - Worker 2 (4 tests) =&gt; 0:00:10.249889\n - Worker 3 (30 tests) =&gt; 0:00:32.730095\n - Worker 4 (51 tests) =&gt; 0:00:26.246044\n - Worker 5 (6 tests) =&gt; 0:00:20.114803\n - Worker 6 (20 tests) =&gt; 0:00:16.290323\n - Worker 7 (27 tests) =&gt; 0:00:17.103827\n</code></pre>"},{"location":"ceph/ceph_rgw/#additional-resources","title":"Additional Resources","text":"<p>A screen recording is available here.</p>"},{"location":"contributing/development_environment/","title":"Development environment","text":"<p>This is a guide for an install_yamls based Adoption environment with network isolation as an alternative to the CRC and Vagrant TripleO Standalone development environment guide.</p> <p>The Adoption development environment utilizes install_yamls for CRC VM creation and for creation of the VM that hosts the original Wallaby OpenStack in Standalone configuration.</p>"},{"location":"contributing/development_environment/#environment-prep","title":"Environment prep","text":"<p>Get install_yamls:</p> <pre><code>git clone https://github.com/openstack-k8s-operators/install_yamls.git\n</code></pre> <p>Install tools for operator development:</p> <pre><code>cd ~/install_yamls/devsetup\nmake download_tools\n</code></pre>"},{"location":"contributing/development_environment/#deployment-of-crc-with-network-isolation","title":"Deployment of CRC with network isolation","text":"<pre><code>cd ~/install_yamls/devsetup\nPULL_SECRET=$HOME/pull-secret.txt CPUS=12 MEMORY=40000 DISK=100 make crc\n\neval $(crc oc-env)\noc login -u kubeadmin -p 12345678 https://api.crc.testing:6443\n\nmake crc_attach_default_interface\n</code></pre>"},{"location":"contributing/development_environment/#development-environment-with-openstack-ironic","title":"Development environment with Openstack ironic","text":"<p>Create the BMaaS network (<code>crc-bmaas</code>) and virtual baremetal nodes controlled by a RedFish BMC emulator.</p> <pre><code>cd ..  # back to install_yamls\nmake nmstate\nmake namespace\ncd devsetup  # back to install_yamls/devsetup\nmake bmaas\n</code></pre> <p>A node definition YAML file to use with the <code>openstack baremetal create &lt;file&gt;.yaml</code> command can be generated for the virtual baremetal nodes by running the <code>bmaas_generate_nodes_yaml</code> make target. Store it in a temp file for later.</p> <pre><code>make bmaas_generate_nodes_yaml | tail -n +2 | tee /tmp/ironic_nodes.yaml\n</code></pre> <p>Set variables to deploy edpm Standalone with additional network (<code>baremetal</code>) and compute driver <code>ironic</code>.</p> <pre><code>cat &lt;&lt; EOF &gt; /tmp/addtional_nets.json\n[\n  {\n    \"type\": \"network\",\n    \"name\": \"crc-bmaas\",\n    \"standalone_config\": {\n      \"type\": \"ovs_bridge\",\n      \"name\": \"baremetal\",\n      \"mtu\": 1500,\n      \"vip\": true,\n      \"ip_subnet\": \"172.20.1.0/24\",\n      \"allocation_pools\": [\n        {\n          \"start\": \"172.20.1.100\",\n          \"end\": \"172.20.1.150\"\n        }\n      ],\n      \"host_routes\": [\n        {\n          \"destination\": \"192.168.130.0/24\",\n          \"nexthop\": \"172.20.1.1\"\n        }\n      ]\n    }\n  }\n]\nEOF\nexport EDPM_COMPUTE_ADDITIONAL_NETWORKS=$(jq -c . /tmp/addtional_nets.json)\nexport STANDALONE_COMPUTE_DRIVER=ironic\nexport NTP_SERVER=pool.ntp.org  # Only neccecary if not on the RedHat network ...\nexport EDPM_COMPUTE_CEPH_ENABLED=false  # Optional\n</code></pre> <p>Use the install_yamls devsetup to create a virtual machine connected to the isolated networks.</p> <p>Create the edpm-compute-0 virtual machine.</p> <pre><code>cd install_yamls/devsetup\nmake standalone\n</code></pre>"},{"location":"contributing/development_environment/#install-the-openstack-k8s-operators-openstack-operator","title":"Install the openstack-k8s-operators (openstack-operator)","text":"<pre><code>cd ..  # back to install_yamls\nmake crc_storage\nmake input\nmake openstack\n</code></pre>"},{"location":"contributing/development_environment/#convenience-steps","title":"Convenience steps","text":"<p>To make our life easier we can copy the deployment passwords we'll be using in the backend services deployment phase of the data plane adoption.</p> <pre><code>scp -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100:/root/tripleo-standalone-passwords.yaml ~/\n</code></pre> <p>If we want to be able to easily run <code>openstack</code> commands from the host without actually installing the package and copying the configuration file from the VM we can create a simple alias:</p> <pre><code>alias openstack=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 OS_CLOUD=standalone openstack\"\n</code></pre>"},{"location":"contributing/development_environment/#route-networks","title":"Route networks","text":"<p>Route VLAN20 to have access to the MariaDB cluster:</p> <pre><code>EDPM_BRIDGE=$(sudo virsh dumpxml edpm-compute-0 | grep -oP \"(?&lt;=bridge=').*(?=')\")\nsudo ip link add link $EDPM_BRIDGE name vlan20 type vlan id 20\nsudo ip addr add dev vlan20 172.17.0.222/24\nsudo ip link set up dev vlan20\n</code></pre>"},{"location":"contributing/development_environment/#snapshotrevert","title":"Snapshot/revert","text":"<p>When the deployment of the Standalone OpenStack is finished, it's a good time to snapshot the machine, so that multiple Adoption attempts can be done without having to deploy from scratch.</p> <pre><code>cd ~/install_yamls/devsetup\nmake standalone_snapshot\n</code></pre> <p>And when you wish to revert the Standalone deployment to the snapshotted state:</p> <pre><code>cd ~/install_yamls/devsetup\nmake standalone_revert\n</code></pre> <p>Similar snapshot could be done for the CRC virtual machine, but the developer environment reset on CRC side can be done sufficiently via the install_yamls <code>*_cleanup</code> targets. This is further detailed in the section: Reset the environment to pre-adoption state</p>"},{"location":"contributing/development_environment/#create-a-workload-to-adopt","title":"Create a workload to adopt","text":""},{"location":"contributing/development_environment/#ironic-steps","title":"Ironic Steps","text":""},{"location":"contributing/development_environment/#enroll-baremetal-nodes-make-bmaas_generate_nodes_yaml-tail-n-2-tee-tmpironic_nodesyaml-scp-i-homeinstall_yamlsoutedpmansibleee-ssh-key-id_rsa-tmpironic_nodesyaml-root192168122100-ssh-i-homeinstall_yamlsoutedpmansibleee-ssh-key-id_rsa-root192168122100-export-os_cloudstandalone-openstack-baremetal-create-rootironic_nodesyaml-export-ironic_python_agent_ramdisk_idopenstack-image-show-deploy-ramdisk-c-id-f-value-export-ironic_python_agent_kernel_idopenstack-image-show-deploy-kernel-c-id-f-value-for-node-in-openstack-baremetal-node-list-c-uuid-f-value-do-openstack-baremetal-node-set-node-driver-info-deploy_ramdiskironic_python_agent_ramdisk_id-driver-info-deploy_kernelironic_python_agent_kernel_id-resource-class-baremetal-property-capabilitiesboot_modeuefi-done-create-a-baremetal-flavor-openstack-flavor-create-baremetal-ram-1024-vcpus-1-disk-15-property-resourcesvcpu0-property-resourcesmemory_mb0-property-resourcesdisk_gb0-property-resourcescustom_baremetal1-property-capabilitiesboot_modeuefi-create-image-imgfedora-cloud-base-38-16x86_64qcow2-urlhttpsdownloadfedoraprojectorgpubfedoralinuxreleases38cloudx86_64imagesimg-curl-o-tmpimg-l-url-disk_formatqemu-img-info-tmpimg-grep-file-format-awk-print-nf-openstack-image-create-container-format-bare-disk-format-disk_format-fedora-cloud-base-38-tmpimg-export-baremetal_nodesopenstack-baremetal-node-list-c-uuid-f-value-manage-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-node-manage-node-done-wait-for-nodes-to-reach-manageable-state-watch-openstack-baremetal-node-list-inspect-baremetal-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-introspection-start-node-done-wait-for-inspection-to-complete-watch-openstack-baremetal-introspection-list-provide-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-node-provide-node-done-wait-for-nodes-to-reach-available-state-watch-openstack-baremetal-node-list-create-an-instance-on-baremetal-openstack-server-show-baremetal-test-openstack-server-create-baremetal-test-flavor-baremetal-image-fedora-cloud-base-38-nic-net-idprovisioning-wait-check-instance-status-and-network-connectivity-openstack-server-show-baremetal-test-ping-c-4-openstack-server-show-baremetal-test-f-json-c-addresses-jq-r-addressesprovisioning0","title":"<pre><code># Enroll baremetal nodes\nmake bmaas_generate_nodes_yaml | tail -n +2 | tee /tmp/ironic_nodes.yaml\nscp -i $HOME/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa /tmp/ironic_nodes.yaml root@192.168.122.100:\nssh -i $HOME/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\n\nexport OS_CLOUD=standalone\nopenstack baremetal create /root/ironic_nodes.yaml\nexport IRONIC_PYTHON_AGENT_RAMDISK_ID=$(openstack image show deploy-ramdisk -c id -f value)\nexport IRONIC_PYTHON_AGENT_KERNEL_ID=$(openstack image show deploy-kernel -c id -f value)\nfor node in $(openstack baremetal node list -c UUID -f value); do\n  openstack baremetal node set $node \\\n    --driver-info deploy_ramdisk=${IRONIC_PYTHON_AGENT_RAMDISK_ID} \\\n    --driver-info deploy_kernel=${IRONIC_PYTHON_AGENT_KERNEL_ID} \\\n    --resource-class baremetal \\\n    --property capabilities='boot_mode:uefi'\ndone\n\n# Create a baremetal flavor\nopenstack flavor create baremetal --ram 1024 --vcpus 1 --disk 15 \\\n  --property resources:VCPU=0 \\\n  --property resources:MEMORY_MB=0 \\\n  --property resources:DISK_GB=0 \\\n  --property resources:CUSTOM_BAREMETAL=1 \\\n  --property capabilities:boot_mode=\"uefi\"\n\n# Create image\nIMG=Fedora-Cloud-Base-38-1.6.x86_64.qcow2\nURL=https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/$IMG\ncurl -o /tmp/${IMG} -L $URL\nDISK_FORMAT=$(qemu-img info /tmp/${IMG} | grep \"file format:\" | awk '{print $NF}')\nopenstack image create --container-format bare --disk-format ${DISK_FORMAT} Fedora-Cloud-Base-38 &lt; /tmp/${IMG}\n\nexport BAREMETAL_NODES=$(openstack baremetal node list -c UUID -f value)\n# Manage nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal node manage $node\ndone\n\n# Wait for nodes to reach \"manageable\" state\nwatch openstack baremetal node list\n\n# Inspect baremetal nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal introspection start $node\ndone\n\n# Wait for inspection to complete\nwatch openstack baremetal introspection list\n\n# Provide nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal node provide $node\ndone\n\n# Wait for nodes to reach \"available\" state\nwatch openstack baremetal node list\n\n# Create an instance on baremetal\nopenstack server show baremetal-test || {\n    openstack server create baremetal-test --flavor baremetal --image Fedora-Cloud-Base-38 --nic net-id=provisioning --wait\n}\n\n# Check instance status and network connectivity\nopenstack server show baremetal-test\nping -c 4 $(openstack server show baremetal-test -f json -c addresses | jq -r .addresses.provisioning[0])\n</code></pre>","text":"<pre><code>export OS_CLOUD=standalone\nsource ~/install_yamls/devsetup/scripts/edpm-deploy-instance.sh\n</code></pre> <p>Confirm the image UUID can be seen in Ceph's images pool. <pre><code>ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 sudo cephadm shell -- rbd -p images ls -l\n</code></pre></p> <p>Create a Cinder volume, a backup from it, and snapshot it. <pre><code>openstack volume create --image cirros --bootable --size 1 disk\nopenstack volume backup create --name backup disk\nopenstack volume snapshot create --volume disk snapshot\n</code></pre></p> <p>Add volume to the test VM <pre><code>openstack server add volume test disk\n</code></pre></p>"},{"location":"contributing/development_environment/#performing-the-data-plane-adoption","title":"Performing the Data Plane Adoption","text":"<p>The development environment is now set up, you can go to the Adoption documentation and perform adoption manually, or run the test suite against your environment.</p>"},{"location":"contributing/development_environment/#reset-the-environment-to-pre-adoption-state","title":"Reset the environment to pre-adoption state","text":"<p>The development environment must be rolled back in case we want to execute another Adoption run.</p> <p>Delete the data-plane and control-plane resources from the CRC vm <pre><code>oc delete osdp openstack\noc delete oscp openstack\n</code></pre></p> <p>Revert the standalone vm to the snapshotted state</p> <p><pre><code>cd ~/install_yamls/devsetup\nmake standalone_revert\n</code></pre> Clean up and initialize the storage PVs in CRC vm <pre><code>cd ..\nmake crc_storage_cleanup\nmake crc_storage\n</code></pre></p>"},{"location":"contributing/development_environment/#experimenting-with-an-additional-compute-node","title":"Experimenting with an additional compute node","text":"<p>The following is not on the critical path of preparing the development environment for Adoption, but it shows how to make the environment work with an additional compute node VM.</p> <p>The remaining steps should be completed on the hypervisor hosting crc and edpm-compute-0.</p>"},{"location":"contributing/development_environment/#deploy-ng-control-plane-with-ceph","title":"Deploy NG Control Plane with Ceph","text":"<p>Export the Ceph configuration from edpm-compute-0 into a secret. <pre><code>SSH=$(ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100)\nKEY=$($SSH \"cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0\")\nCONF=$($SSH \"cat /etc/ceph/ceph.conf | base64 -w 0\")\n\ncat &lt;&lt;EOF &gt; ceph_secret.yaml\napiVersion: v1\ndata:\n  ceph.client.openstack.keyring: $KEY\n  ceph.conf: $CONF\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\ntype: Opaque\nEOF\n\noc create -f ceph_secret.yaml\n</code></pre> Deploy the NG control plane with Ceph as backend for Glance and Cinder. As described in the install_yamls README, use the sample config located at https://github.com/openstack-k8s-operators/openstack-operator/blob/main/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml but make sure to replace the <code>_FSID_</code> in the sample with the one from the secret created in the previous step. <pre><code>curl -o /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml https://raw.githubusercontent.com/openstack-k8s-operators/openstack-operator/main/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\nFSID=$(oc get secret ceph-conf-files -o json | jq -r '.data.\"ceph.conf\"' | base64 -d | grep fsid | sed -e 's/fsid = //') &amp;&amp; echo $FSID\nsed -i \"s/_FSID_/${FSID}/\" /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\noc apply -f /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\n</code></pre></p> <p>A NG control plane which uses the same Ceph backend should now be functional. If you create a test image on the NG system to confirm it works from the configuration above, be sure to read the warning in the next section.</p> <p>Before beginning adoption testing or development you may wish to deploy an EDPM node as described in the following section.</p>"},{"location":"contributing/development_environment/#warning-about-two-openstacks-and-one-ceph","title":"Warning about two OpenStacks and one Ceph","text":"<p>Though workloads can be created in the NG deployment to test, be careful not to confuse them with workloads from the Wallaby cluster to be migrated. The following scenario is now possible.</p> <p>A Glance image exists on the Wallaby OpenStack to be adopted. <pre><code>[stack@standalone standalone]$ export OS_CLOUD=standalone\n[stack@standalone standalone]$ openstack image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 33a43519-a960-4cd0-a593-eca56ee553aa | cirros | active |\n+--------------------------------------+--------+--------+\n[stack@standalone standalone]$\n</code></pre> If you now create an image with the NG cluster, then a Glance image will exsit on the NG OpenStack which will adopt the workloads of the wallaby. <pre><code>[fultonj@hamfast ng]$ export OS_CLOUD=default\n[fultonj@hamfast ng]$ export OS_PASSWORD=12345678\n[fultonj@hamfast ng]$ openstack image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 4ebccb29-193b-4d52-9ffd-034d440e073c | cirros | active |\n+--------------------------------------+--------+--------+\n[fultonj@hamfast ng]$\n</code></pre> Both Glance images are stored in the same Ceph pool. <pre><code>ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 sudo cephadm shell -- rbd -p images ls -l\nInferring fsid 7133115f-7751-5c2f-88bd-fbff2f140791\nUsing recent ceph image quay.rdoproject.org/tripleowallabycentos9/daemon@sha256:aa259dd2439dfaa60b27c9ebb4fb310cdf1e8e62aa7467df350baf22c5d992d8\nNAME                                       SIZE     PARENT  FMT  PROT  LOCK\n33a43519-a960-4cd0-a593-eca56ee553aa         273 B            2\n33a43519-a960-4cd0-a593-eca56ee553aa@snap    273 B            2  yes\n4ebccb29-193b-4d52-9ffd-034d440e073c       112 MiB            2\n4ebccb29-193b-4d52-9ffd-034d440e073c@snap  112 MiB            2  yes\n</code></pre> However, as far as each Glance service is concerned each has one image. Thus, in order to avoid confusion during adoption the test Glance image on the NG OpenStack should be deleted. <pre><code>openstack image delete 4ebccb29-193b-4d52-9ffd-034d440e073c\n</code></pre> Connecting the NG OpenStack to the existing Ceph cluster is part of the adoption procedure so that the data migration can be minimized but understand the implications of the above example.</p>"},{"location":"contributing/development_environment/#deploy-edpm-compute-1","title":"Deploy edpm-compute-1","text":"<p>edpm-compute-0 is not available as a standard EDPM system to be managed by edpm-ansible or dataplane-operator because it hosts the wallaby deployment which will be adopted and after adoption it will only host the Ceph server.</p> <p>Use the install_yamls devsetup to create additional virtual machines and be sure that the <code>EDPM_COMPUTE_SUFFIX</code> is set to <code>1</code> or greater. Do not set <code>EDPM_COMPUTE_SUFFIX</code> to <code>0</code> or you could delete the Wallaby system created in the previous section.</p> <p>When deploying EDPM nodes add an <code>extraMounts</code> like the following in the <code>OpenStackDataPlaneNodeSet</code> CR <code>nodeTemplate</code> so that they will be configured to use the same Ceph cluster.</p> <pre><code>    edpm-compute:\n      nodeTemplate:\n        extraMounts:\n        - extraVolType: Ceph\n          volumes:\n          - name: ceph\n            secret:\n              secretName: ceph-conf-files\n          mounts:\n          - name: ceph\n            mountPath: \"/etc/ceph\"\n            readOnly: true\n</code></pre> <p>A NG data plane which uses the same Ceph backend should now be functional. Be careful about not confusing new workloads to test the NG OpenStack with the Wallaby OpenStack as described in the previous section.</p>"},{"location":"contributing/development_environment/#begin-adoption-testing-or-development","title":"Begin Adoption Testing or Development","text":"<p>We should now have:</p> <ul> <li>An NG glance service based on Antelope running on CRC</li> <li>An TripleO-deployed glance serviced running on edpm-compute-0</li> <li>Both services have the same Ceph backend</li> <li>Each service has their own independent database</li> </ul> <p>An environment above is assumed to be available in the Glance Adoption documentation. You may now follow other Data Plane Adoption procedures described in the documentation. The same pattern can be applied to other services.</p>"},{"location":"contributing/documentation/","title":"Contributing to documentation","text":""},{"location":"contributing/documentation/#rendering-documentation-locally","title":"Rendering documentation locally","text":"<p>Install docs build requirements into virtualenv:</p> <pre><code>python3 -m venv local/docs-venv\nsource local/docs-venv/bin/activate\npip install -r docs/doc_requirements.txt\n</code></pre> <p>Serve docs site on localhost:</p> <pre><code>mkdocs serve\n</code></pre> <p>Click the link it outputs. As you save changes to files modified in your editor, the browser will automatically show the new content.</p>"},{"location":"contributing/documentation/#patterns-and-tips-for-contributing-to-documentation","title":"Patterns and tips for contributing to documentation","text":"<ul> <li> <p>Pages concerning individual components/services should make sense in   the context of the broader adoption procedure. While adopting a   service in isolation is an option for developers, let's write the   documentation with the assumption the adoption procedure is being   done in full, going step by step (one doc after another).</p> </li> <li> <p>The procedure should be written with production use in mind. This   repository could be used as a starting point for product   technical documentation. We should not tie the documentation to   something that wouldn't translate well from dev envs to production.</p> <ul> <li>This includes not assuming that the source environment is   Standalone, and the destination is CRC. We can provide examples for   Standalone/CRC, but it should be possible to use the procedure   with fuller environments in a way that is obvious from the docs.</li> </ul> </li> <li> <p>If possible, try to make code snippets copy-pastable. Use shell   variables if the snippets should be parametrized. Use <code>oc</code> rather   than <code>kubectl</code> in snippets.</p> </li> <li> <p>Focus on the \"happy path\" in the docs as much as possible,   troubleshooting info can go into the Troubleshooting page, or   alternatively a troubleshooting section at the end of the document,   visibly separated from the main procedure.</p> </li> <li> <p>The full procedure will inevitably happen to be quite long, so let's   try to be concise in writing to keep the docs consumable (but not to   a point of making things difficult to understand or omitting   important things).</p> </li> <li> <p>A bash alias can be created for long command however when implementing   them in the test roles you should transform them to avoid command not   found errors.   From:   <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\nopenstack endpoint list | grep network\n</code></pre>   TO:   <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\n${BASH_ALIASES[openstack]} endpoint list | grep network\n</code></pre></p> </li> </ul>"},{"location":"contributing/tests/","title":"Tests","text":""},{"location":"contributing/tests/#test-suite-information","title":"Test suite information","text":"<p>The adoption docs repository also includes a test suite for Adoption. There are targets in the Makefile which can be used to execute the test suite:</p> <ul> <li> <p><code>test-minimal</code> - a minimal test scenario, the eventual set of   services in this scenario should be the \"core\" services needed to   launch a VM. This scenario assumes local storage backend for   services like Glance and Cinder.</p> </li> <li> <p><code>test-with-ceph</code> - like 'minimal' but with Ceph storage backend for   Glance and Cinder.</p> </li> </ul>"},{"location":"contributing/tests/#configuring-the-test-suite","title":"Configuring the test suite","text":"<ul> <li> <p>Create <code>tests/vars.yaml</code> and <code>tests/secrets.yaml</code> by copying the   included samples (<code>tests/vars.sample.yaml</code>,   <code>tests/secrets.sample.yaml</code>).</p> </li> <li> <p>Walk through the <code>tests/vars.yaml</code> and <code>tests/secrets.yaml</code> files   and see if you need to edit any values. If you are using the   documented development environment, majority of the defaults should   work out of the box. The comments in the YAML files will guide you   regarding the expected values. You may want to double check that   these variables suit your environment:</p> </li> <li> <p><code>install_yamls_path</code></p> </li> <li> <p><code>tripleo_passwords</code></p> </li> <li> <p><code>controller*_ssh</code></p> </li> <li> <p><code>edpm_privatekey_path</code></p> </li> <li> <p><code>edpm_chrony_ntp_servers</code></p> </li> </ul>"},{"location":"contributing/tests/#running-the-tests","title":"Running the tests","text":"<p>The interface between the execution infrastructure and the test suite is an Ansible inventory and variables files. Inventory and variable samples are provided. To run the tests, follow this procedure:</p> <ul> <li>Install dependencies and create a venv:</li> </ul> <pre><code>sudo dnf -y install python-devel\npython3 -m venv venv\nsource venv/bin/activate\npip install openstackclient osc_placement jmespath\nansible-galaxy collection install community.general\n</code></pre> <ul> <li>Run <code>make test-with-ceph</code> (the documented development environment   does include Ceph).</li> </ul> <p>If you are using Ceph-less environment, you should run <code>make   test-minimal</code>.</p>"},{"location":"contributing/tests/#making-patches-to-the-test-suite","title":"Making patches to the test suite","text":"<p>Please be aware of the following when changing the test suite:</p> <ul> <li>The test suite should follow the docs as much as possible.</li> </ul> <p>The purpose of the test suite is to verify what the user would run   if they were following the docs. We don't want to loosely rewrite   the docs into Ansible code following Ansible best practices. We want   to test the exact same bash commands/snippets that are written in   the docs. This often means that we should be using the <code>shell</code>   module and do a verbatim copy/paste from docs, instead of using the   best Ansible module for the task at hand.</p>"},{"location":"openstack/backend_services_deployment/","title":"Backend services deployment","text":"<p>The following instructions create OpenStackControlPlane CR with basic backend services deployed, and all the OpenStack services disabled. This will be the foundation of the podified control plane.</p> <p>In subsequent steps, we'll import the original databases and then add podified OpenStack control plane services.</p>"},{"location":"openstack/backend_services_deployment/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>The cloud which we want to adopt is up and running. It's on   OpenStack Wallaby release.</p> </li> <li> <p>The <code>openstack-operator</code> is deployed, but <code>OpenStackControlPlane</code> is   not deployed.</p> </li> </ul> <p>For developer/CI environments, the openstack operator can be deployed   by running <code>make openstack</code> inside   install_yamls   repo.</p> <p>For production environments, the deployment method will likely be   different.</p> <ul> <li>There are free PVs available to be claimed (for MariaDB and RabbitMQ).</li> </ul> <p>For developer/CI environments driven by install_yamls, make sure   you've run <code>make crc_storage</code>.</p>"},{"location":"openstack/backend_services_deployment/#variables","title":"Variables","text":"<ul> <li>Set the desired admin password for the podified deployment. This can   be the original deployment's admin password or something else.</li> </ul> <pre><code>ADMIN_PASSWORD=SomePassword\n</code></pre> <p>To use the existing OpenStack deployment password:   <pre><code>ADMIN_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' AdminPassword:' | awk -F ': ' '{ print $2; }')\n</code></pre></p> <ul> <li>Set service password variables to match the original deployment.   Database passwords can differ in podified environment, but   synchronizing the service account passwords is a required step.</li> </ul> <p>E.g. in developer environments with TripleO Standalone, the   passwords can be extracted like this:</p> <pre><code>CINDER_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' CinderPassword:' | awk -F ': ' '{ print $2; }')\nGLANCE_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' GlancePassword:' | awk -F ': ' '{ print $2; }')\nIRONIC_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' IronicPassword:' | awk -F ': ' '{ print $2; }')\nNEUTRON_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NeutronPassword:' | awk -F ': ' '{ print $2; }')\nNOVA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NovaPassword:' | awk -F ': ' '{ print $2; }')\nOCTAVIA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' OctaviaPassword:' | awk -F ': ' '{ print $2; }')\nPLACEMENT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' PlacementPassword:' | awk -F ': ' '{ print $2; }')\n</code></pre>"},{"location":"openstack/backend_services_deployment/#pre-checks","title":"Pre-checks","text":""},{"location":"openstack/backend_services_deployment/#procedure-backend-services-deployment","title":"Procedure - backend services deployment","text":"<ul> <li>Make sure you are using the OpenShift namespace where you want the   podified control plane deployed:</li> </ul> <pre><code>oc project openstack\n</code></pre> <ul> <li>Create OSP secret.</li> </ul> <p>The procedure for this will vary, but in developer/CI environments   we use install_yamls:</p> <pre><code># in install_yamls\nmake input\n</code></pre> <ul> <li>If the <code>$ADMIN_PASSWORD</code> is different than the already set password   in <code>osp-secret</code>, amend the <code>AdminPassword</code> key in the <code>osp-secret</code>   correspondingly:</li> </ul> <pre><code>oc set data secret/osp-secret \"AdminPassword=$ADMIN_PASSWORD\"\n</code></pre> <ul> <li>Set service account passwords in <code>osp-secret</code> to match the service   account passwords from the original deployment:</li> </ul> <pre><code>oc set data secret/osp-secret \"CinderPassword=$CINDER_PASSWORD\"\noc set data secret/osp-secret \"GlancePassword=$GLANCE_PASSWORD\"\noc set data secret/osp-secret \"IronicPassword=$IRONIC_PASSWORD\"\noc set data secret/osp-secret \"NeutronPassword=$NEUTRON_PASSWORD\"\noc set data secret/osp-secret \"NovaPassword=$NOVA_PASSWORD\"\noc set data secret/osp-secret \"OctaviaPassword=$OCTAVIA_PASSWORD\"\noc set data secret/osp-secret \"PlacementPassword=$PLACEMENT_PASSWORD\"\n</code></pre> <ul> <li>Deploy OpenStackControlPlane. Make sure to only enable DNS,   MariaDB, Memcached, and RabbitMQ services. All other services must   be disabled.</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n\n  cinder:\n    enabled: false\n    template:\n      cinderAPI: {}\n      cinderScheduler: {}\n      cinderBackup: {}\n      cinderVolumes: {}\n\n  dns:\n    template:\n      override:\n        service:\n          metadata:\n            annotations:\n              metallb.universe.tf/address-pool: ctlplane\n              metallb.universe.tf/allow-shared-ip: ctlplane\n              metallb.universe.tf/loadBalancerIPs: 192.168.122.80\n          spec:\n            type: LoadBalancer\n      options:\n      - key: server\n        values:\n        - 192.168.122.1\n      replicas: 1\n\n  glance:\n    enabled: false\n    template:\n      glanceAPIInternal: {}\n      glanceAPIExternal: {}\n\n  horizon:\n    enabled: false\n    template: {}\n\n  ironic:\n    enabled: false\n    template:\n      ironicConductors: []\n\n  keystone:\n    enabled: false\n    template: {}\n\n  manila:\n    enabled: false\n    template:\n      manilaAPI: {}\n      manilaScheduler: {}\n      manilaShares: {}\n\n  mariadb:\n    templates:\n      openstack:\n        storageRequest: 500M\n      openstack-cell1:\n        storageRequest: 500M\n\n  memcached:\n    enabled: true\n    templates:\n      memcached:\n        replicas: 1\n\n  neutron:\n    enabled: false\n    template: {}\n\n  nova:\n    enabled: false\n    template: {}\n\n  ovn:\n    enabled: false\n    template:\n      ovnController:\n        external-ids:\n          system-id: \"random\"\n          ovn-bridge: \"br-int\"\n          ovn-encap-type: \"geneve\"\n\n  placement:\n    enabled: false\n    template: {}\n\n  rabbitmq:\n    templates:\n      rabbitmq:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.85\n            spec:\n              type: LoadBalancer\n      rabbitmq-cell1:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.86\n            spec:\n              type: LoadBalancer\n\n  telemetry:\n    enabled: false\n    template: {}\nEOF\n</code></pre>"},{"location":"openstack/backend_services_deployment/#post-checks","title":"Post-checks","text":"<ul> <li>Check that MariaDB is running.</li> </ul> <pre><code>oc get pod mariadb-openstack -o jsonpath='{.status.phase}{\"\\n\"}'\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/","title":"Ceph backend configuration (if applicable)","text":"<p>If the original deployment uses a Ceph storage backend for any service (e.g. Glance, Cinder, Nova), the same backend must be used in the adopted deployment and CRs must be configured accordingly.</p>"},{"location":"openstack/ceph_backend_configuration/#prerequisites","title":"Prerequisites","text":"<ul> <li>The <code>OpenStackControlPlane</code> CR must already exist.</li> </ul>"},{"location":"openstack/ceph_backend_configuration/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>CEPH_SSH=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\"\nCEPH_KEY=$($CEPH_SSH \"cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0\")\nCEPH_CONF=$($CEPH_SSH \"cat /etc/ceph/ceph.conf | base64 -w 0\")\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/#procedure-ceph-backend-configuration","title":"Procedure - Ceph backend configuration","text":"<p>Create the <code>ceph-conf-files</code> secret, containing Ceph configuration:</p> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\ndata:\n  ceph.client.openstack.keyring: $CEPH_KEY\n  ceph.conf: $CEPH_CONF\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\ntype: Opaque\nEOF\n</code></pre> <p>The content of the file should look something like this:</p> <pre><code>---\napiVersion: v1\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\nstringData:\n  ceph.client.openstack.keyring: |\n    [client.openstack]\n        key = &lt;secret key&gt;\n        caps mgr = \"allow *\"\n        caps mon = \"profile rbd\"\n        caps osd = \"profile rbd pool=images\"\n  ceph.conf: |\n    [global]\n    fsid = 7a1719e8-9c59-49e2-ae2b-d7eb08c695d4\n    mon_host = 10.1.1.2,10.1.1.3,10.1.1.4\n</code></pre> <p>Configure <code>extraMounts</code> within the <code>OpenStackControlPlane</code> CR:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  extraMounts:\n    - name: v1\n      region: r1\n      extraVol:\n        - propagation:\n          - CinderVolume\n          - CinderBackup\n          - GlanceAPI\n          extraVolType: Ceph\n          volumes:\n          - name: ceph\n            projected:\n              sources:\n              - secret:\n                  name: ceph-conf-files\n          mounts:\n          - name: ceph\n            mountPath: \"/etc/ceph\"\n            readOnly: true\n'\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/#getting-ceph-fsid","title":"Getting Ceph FSID","text":"<p>Configuring some OpenStack services to use Ceph backend may require the FSID value. You can fetch the value from the config like so:</p> <pre><code>CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data.\"ceph.conf\"' | base64 -d | grep fsid | sed -e 's/fsid = //')\n</code></pre>"},{"location":"openstack/cinder_adoption/","title":"Cinder adoption","text":"<p>Adopting a director deployed Cinder service into OpenStack may require some thought because it's not always a simple process.</p> <p>Usually the adoption process entails:</p> <ul> <li>Checking existing limitations.</li> <li>Considering the placement of the cinder services.</li> <li>Preparing the OpenShift nodes where volume and backup services will run.</li> <li>Crafting the manifest based on the existing <code>cinder.conf</code> file.</li> <li>Deploying Cinder.</li> <li>Validating the new deployment.</li> </ul> <p>This guide provides necessary knowledge to complete these steps in most situations, but it still requires knowledge on how OpenStack services work and the structure of a Cinder configuration file.</p>"},{"location":"openstack/cinder_adoption/#limitations","title":"Limitations","text":"<p>There are currently some limitations that are worth highlighting; some are related to this guideline while some to the operator:</p> <ul> <li> <p>There is no global <code>nodeSelector</code> for all cinder volumes, so it needs to be specified per backend.  This may change in the future.</p> </li> <li> <p>There is no global <code>customServiceConfig</code> or <code>customServiceConfigSecrets</code> for all cinder volumes, so it needs to be specified per backend.  This may change in the future.</p> </li> <li> <p>Adoption of LVM backends, where the volume data is stored in the compute nodes, is not currently being documented in this process. It may get documented in the future.</p> </li> <li> <p>Support for Cinder backends that require kernel modules not included in RHEL has not been tested in Operator deployed OpenStack so it is not documented in this guide.</p> </li> <li> <p>Adoption of DCN/Edge deployment is not currently described in this guide.</p> </li> </ul>"},{"location":"openstack/cinder_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>Previous Adoption steps completed. Notably, cinder service must have been stopped and the service databases must already be imported into the podified MariaDB.</p> </li> <li> <p>Storage network has been properly configured on the OpenShift cluster.</p> </li> </ul>"},{"location":"openstack/cinder_adoption/#variables","title":"Variables","text":"<p>No new environmental variables need to be defined, though we use the <code>CONTROLLER1_SSH</code> that was defined in a previous step for the pre-checks.</p>"},{"location":"openstack/cinder_adoption/#pre-checks","title":"Pre-checks","text":"<p>We are going to need the contents of <code>cinder.conf</code>, so we may want to download it to have it locally accessible:</p> <pre><code>$CONTROLLER1_SSH cat /var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf &gt; cinder.conf\n</code></pre>"},{"location":"openstack/cinder_adoption/#prepare-openshift","title":"Prepare OpenShift","text":"<p>As explained the planning section before deploying OpenStack in OpenShift we need to ensure that the networks are ready, that we have decided the node selection, and also make sure any necessary changes to the OpenShift nodes have been made.  For Cinder volume and backup services all these 3 must be carefully considered.</p>"},{"location":"openstack/cinder_adoption/#node-selection","title":"Node Selection","text":"<p>We may need, or want, to restrict the OpenShift nodes where cinder volume and backup services can run.</p> <p>The best example of when we need to do node selection for a specific cinder service in when we deploy Cinder with the LVM driver. In that scenario the LVM data where the volumes are stored only exists in a specific host, so we need to pin the cinder-volume service to that specific OpenShift node.  Running the service on any other OpenShift node would not work.  Since <code>nodeSelector</code> only works on labels we cannot use the OpenShift host node name to restrict the LVM backend and we'll need to identify it using a unique label, an existing or new one:</p> <pre><code>$ oc label nodes worker0 lvm=cinder-volumes\n</code></pre> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  cinder:\n    enabled: true\n    template:\n      cinderVolumes:\n        lvm-iscsi:\n          nodeSelector:\n            lvm: cinder-volumes\n&lt; . . . &gt;\n</code></pre> <p>As mentioned in the Node Selector guide, an example where we need to use labels is when using FC storage and we don't have HBA cards in all our OpenShift nodes. In this scenario we would need to restrict all the cinder volume backends (not only the FC one) as well as the backup services.</p> <p>Depending on the cinder backends, their configuration, and the usage of Cinder, we can have network intensive cinder volume services with lots of I/O as well as cinder backup services that are not only network intensive but also memory and CPU intensive.  This may be a concern for the OpenShift human operators, and they may want to use the <code>nodeSelector</code> to prevent these service from interfering with their other OpenShift workloads</p> <p>Please make sure to read the Nodes Selector guide before continuing, as we'll be referring to some of the concepts explained there in the following sections.</p> <p>When selecting the nodes where cinder volume is going to run please remember that cinder-volume may also use local storage when downloading a glance image for the create volume from image operation, and it can require a considerable amount of space when having concurrent operations and not using cinder volume cache.</p> <p>If we don't have nodes with enough local disk space for the temporary images we can use a remote NFS location for the images. This is something that we had to manually setup in Director deployments, but with operators we can easily do it automatically using the extra volumes feature ()<code>extraMounts</code>.</p>"},{"location":"openstack/cinder_adoption/#transport-protocols","title":"Transport protocols","text":"<p>Due to the specifics of the storage transport protocols some changes may be required on the OpenShift side, and although this is something that must be documented by the Vendor here wer are going to provide some generic instructions that can serve as a guide for the different transport protocols.</p> <p>Check the backend sections in our <code>cinder.conf</code> file that are listed in the <code>enabled_backends</code> configuration option to figure out the transport storage protocol used by the backend.</p> <p>Depending on the backend we can find the transport protocol:</p> <ul> <li> <p>Looking at the <code>volume_driver</code> configuration option, as it may contain the   protocol itself: RBD, iSCSI, FC...</p> </li> <li> <p>Looking at the <code>target_protocol</code> configuration option</p> </li> </ul> <p>Warning: Any time a <code>MachineConfig</code> is used to make changes to OpenShift nodes the node will reboot!!  Act accordingly.</p>"},{"location":"openstack/cinder_adoption/#nfs","title":"NFS","text":"<p>There's nothing to do for NFS. OpenShift can connect to NFS backends without any additional changes.</p>"},{"location":"openstack/cinder_adoption/#rbdceph","title":"RBD/Ceph","text":"<p>There's nothing to do for RBD/Ceph in terms of preparing the nodes, OpenShift can connect to Ceph backends without any additional changes. Credentials and configuration files will need to be provided to the services though.</p>"},{"location":"openstack/cinder_adoption/#iscsi","title":"iSCSI","text":"<p>Connecting to iSCSI volumes requires that the iSCSI initiator is running on the OpenShift hosts hosts where volume and backup services are going to run, because the Linux Open iSCSI initiator doesn't currently support network namespaces, so we must only run 1 instance of the service for the normal OpenShift usage, plus the OpenShift CSI plugins, plus the OpenStack services.</p> <p>If we are not already running <code>iscsid</code> on the OpenShift nodes then we'll need to apply a <code>MachineConfig</code> similar to this one:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-enable-iscsid\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    systemd:\n      units:\n      - enabled: true\n        name: iscsid.service\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>For production deployments using iSCSI volumes we always recommend setting up multipathing, please look at the multipathing section to see how to configure it.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for iSCSI.</p>"},{"location":"openstack/cinder_adoption/#fc","title":"FC","text":"<p>There's nothing to do for FC volumes to work, but the cinder volume and cinder backup services need to run in an OpenShift host that has HBAs, so if there are nodes that don't have HBAs then we'll need to use labels to restrict where these services can run, as mentioned in the node labels section.</p> <p>This also means that for virtualized OpenShift clusters using FC we'll need to expose the host's HBAs inside the VM.</p> <p>For production deployments using FC volumes we always recommend setting up multipathing, please look at the multipathing section to see how to configure it.</p>"},{"location":"openstack/cinder_adoption/#nvme-of","title":"NVMe-oF","text":"<p>Connecting to NVMe-oF volumes requires that the nvme kernel modules are loaded on the OpenShift hosts.</p> <p>If we are not already loading the <code>nvme-fabrics</code> module on the OpenShift nodes where volume and backup services are going to run then we'll need to apply a <code>MachineConfig</code> similar to this one:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-load-nvme-fabrics\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    storage:\n      files:\n        - path: /etc/modules-load.d/nvme_fabrics.conf\n          overwrite: false\n          # Mode must be decimal, this is 0644\n          mode: 420\n          user:\n            name: root\n          group:\n            name: root\n          contents:\n            # Source can be a http, https, tftp, s3, gs, or data as defined in rfc2397.\n            # This is the rfc2397 text/plain string format\n            source: data:,nvme-fabrics\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>We are only loading the <code>nvme-fabrics</code> module because it takes care of loading the transport specific modules (tcp, rdma, fc) as needed.</p> <p>For production deployments using NVMe-oF volumes we always recommend using multipathing. For NVMe-oF volumes OpenStack uses native multipathing, called ANA.</p> <p>Once the OpenShift nodes have rebooted and are loading the <code>nvme-fabrics</code> module we can confirm that the Operating System is configured and supports ANA by checking on the host:</p> <pre><code>cat /sys/module/nvme_core/parameters/multipath\n</code></pre> <p>Attention: ANA doesn't use the Linux Multipathing Device Mapper, but the *current OpenStack code requires <code>multipathd</code> on compute nodes to be running for Nova to be able to use multipathing, so please remember to follow the multipathing part for compute nodes on the multipathing section.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for NVMe-oF.</p>"},{"location":"openstack/cinder_adoption/#multipathing","title":"Multipathing","text":"<p>For iSCSI and FC protocols we always recommend using multipathing, which has 4 parts:</p> <ul> <li>Prepare the OpenShift hosts</li> <li>Configure the Cinder services</li> <li>Prepare the Nova computes</li> <li>Configure the Nova service</li> </ul> <p>To prepare the OpenShift hosts we need to ensure that the Linux Multipath Device Mapper is configured and running on the OpenShift hosts, and we do that using <code>MachineConfig</code> like this one:</p> <pre><code># Includes the /etc/multipathd.conf contents and the systemd unit changes\napiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-enable-multipathd\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    storage:\n      files:\n        - path: /etc/multipath.conf\n          overwrite: false\n          # Mode must be decimal, this is 0600\n          mode: 384\n          user:\n            name: root\n          group:\n            name: root\n          contents:\n            # Source can be a http, https, tftp, s3, gs, or data as defined in rfc2397.\n            # This is the rfc2397 text/plain string format\n            source: data:,defaults%20%7B%0A%20%20user_friendly_names%20no%0A%20%20recheck_wwid%20yes%0A%20%20skip_kpartx%20yes%0A%20%20find_multipaths%20yes%0A%7D%0A%0Ablacklist%20%7B%0A%7D\n    systemd:\n      units:\n      - enabled: true\n        name: multipathd.service\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>To configure the cinder services to use multipathing we need to enable the <code>use_multipath_for_image_xfer</code> configuration option in all the backend sections and in the <code>[DEFAULT]</code> section for the backup service, but in Podified deployments we don't need to worry about it, because that's the default. So as long as we don't override it setting <code>use_multipath_for_image_xfer = false</code> then multipathing will work as long as the service is running on the OpenShift host.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for Multipathing once it's implemented.</p>"},{"location":"openstack/cinder_adoption/#configurations","title":"Configurations","text":"<p>As described in the planning Cinder is configured using configuration snippets instead of using obscure configuration parameters defined by the installer.</p> <p>The recommended way to deploy Cinder volume backends has changed to remove old limitations, add flexibility, and improve operations in general.</p> <p>When deploying with Director we used to run a single Cinder volume service with all our backends (each backend would run on its own process), and even though that way of deploying is still supported, we don't recommend it. We recommend using a volume service per backend since it's a superior deployment model.</p> <p>So for an LVM and a Ceph backend we would have 2 entries in <code>cinderVolume</code> and, as mentioned in the limitations section, we cannot set global defaults for all volume services, so we would have to define it for each of them, like this:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    enabled: true\n    template:\n      cinderVolume:\n        lvm:\n          customServiceConfig: |\n            [DEFAULT]\n            debug = True\n            [lvm]\n&lt; . . . &gt;\n        ceph:\n          customServiceConfig: |\n            [DEFAULT]\n            debug = True\n            [ceph]\n&lt; . . . &gt;\n</code></pre> <p>Reminder that for volume backends that have sensitive information using <code>Secret</code> and the <code>customServiceConfigSecrets</code> key is the recommended way to go.</p>"},{"location":"openstack/cinder_adoption/#prepare-the-configuration","title":"Prepare the configuration","text":"<p>For adoption instead of using a whole deployment manifest we'll use a targeted patch, like we did with other services, and in this patch we will enable the different cinder services with their specific configurations.</p> <p>WARNING: Check that all configuration options are still valid for the new OpenStack version, since configuration options may have been deprecated, removed, or added. This applies to both backend driver specific configuration options and other generic options.</p> <p>There are 2 ways to prepare a cinder configuration for adoption, tailor-making it or doing it quick and dirty. There is no difference in how Cinder will operate with both methods, so we are free to chose, though we recommend tailor-making it whenever possible.</p> <p>The high level explanation of the tailor-made approach is:</p> <ol> <li> <p>Determine what part of the configuration is generic for all the cinder services and remove anything that would change when deployed in OpenShift, like the <code>connection</code> in the <code>[dabase]</code> section, the <code>transport_url</code> and <code>log_dir</code> in <code>[DEFAULT]</code>, the whole <code>[coordination]</code> section.  This configuration goes into the <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>) at the <code>cinder: template:</code> level.</p> </li> <li> <p>Determine if there's any scheduler specific configuration and add it to the <code>customServiceConfig</code> section in <code>cinder: template: cinderScheduler</code>.</p> </li> <li> <p>Determine if there's any API specific configuration and add it to the <code>customServiceConfig</code> section in <code>cinder: template: cinderAPI</code>.</p> </li> <li> <p>If we have cinder backup deployed, then we'll get the cinder backup relevant configuration options and add them to <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>) at the <code>cinder: template: cinderBackup:</code> level. We should remove the <code>host</code> configuration in the <code>[DEFAULT]</code> section to facilitate supporting multiple replicas in the future.</p> </li> <li> <p>Determine the individual volume backend configuration for each of the drivers. The configuration will not only be the specific driver section, it should also include the <code>[backend_defaults]</code> section and FC zoning sections is they are being used, because the cinder operator doesn't support a <code>customServiceConfig</code> section global for all volume services.  Each backend would have its own section under <code>cinder: template: cinderVolumes</code> and the configuration would go in <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>).</p> </li> <li> <p>Check if any of the cinder volume drivers being used requires a custom vendor image. If they do, find the location of the image in the vendor's instruction available in the w OpenStack Cinder ecosystem page and add it under the specific's driver section using the <code>containerImage</code> key. For example, if we had a Pure Storage array and the driver was already certified for OSP18, then we would have something like this:</p> </li> </ol> <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderVolume:\n        pure:\n          containerImage: registry.connect.redhat.com/purestorage/openstack-cinder-volume-pure-rhosp-18-0'\n          customServiceConfigSecrets:\n            - openstack-cinder-pure-cfg\n&lt; . . . &gt;\n</code></pre> <ol> <li>External files: Cinder services sometimes use external files, for example for a custom policy, or to store credentials, or SSL CA bundles to connect to a storage array, and we need to make those files available to the right containers. To achieve this we'll use <code>Secrets</code> or <code>ConfigMap</code> to store the information in OpenShift and then the <code>extraMounts</code> key. For example, for the Ceph credentials stored in a <code>Secret</code> called <code>ceph-conf-files</code> we would patch the top level <code>extraMounts</code> in <code>OpenstackControlPlane</code>:</li> </ol> <p><pre><code>spec:\n  extraMounts:\n  - extraVol:\n    - extraVolType: Ceph\n      mounts:\n      - mountPath: /etc/ceph\n        name: ceph\n        readOnly: true\n      propagation:\n      - CinderVolume\n      - CinderBackup\n      - Glance\n      volumes:\n      - name: ceph\n        projected:\n          sources:\n          - secret:\n              name: ceph-conf-files\n</code></pre>    But for a service specific one, like the API policy, we would do it directly    on the service itself, in this example we include the cinder API    configuration that references the policy we are adding from a <code>ConfigMap</code>    called <code>my-cinder-conf</code> that has a key <code>policy</code> with the contents of the    policy:    <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderAPI:\n        customServiceConfig: |\n           [oslo_policy]\n           policy_file=/etc/cinder/api/policy.yaml\n      extraMounts:\n      - extraVol:\n        - extraVolType: Ceph\n          mounts:\n          - mountPath: /etc/cinder/api\n            name: policy\n            readOnly: true\n          propagation:\n          - CinderAPI\n          volumes:\n          - name: policy\n            projected:\n              sources:\n              - configMap:\n                  name: my-cinder-conf\n                  items:\n                    - key: policy\n                      path: policy.yaml\n</code></pre></p> <p>The quick and dirty process is more straightforward:</p> <ol> <li> <p>Create an agnostic configuration file removing any specifics from the old deployment's <code>cinder.conf</code> file, like the <code>connection</code> in the <code>[dabase]</code> section, the <code>transport_url</code> and <code>log_dir</code> in <code>[DEFAULT]</code>, the whole <code>[coordination]</code> section, etc..</p> </li> <li> <p>Assuming the configuration has sensitive information, drop the modified contents of the whole file into a <code>Secret</code>.</p> </li> <li> <p>Reference this secret in all the services, creating a cinder volumes section for each backend and just adding the respective <code>enabled_backends</code> option.</p> </li> <li> <p>Add external files as mentioned in the last bullet of the tailor-made configuration explanation.</p> </li> </ol> <p>Example of what the quick and dirty configuration patch would look like:</p> <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderAPI:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderScheduler:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderBackup:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderVolume:\n        lvm1:\n          customServiceConfig: |\n            [DEFAULT]\n            enabled_backends = lvm1\n          customServiceConfigSecrets:\n            - cinder-conf\n        lvm2:\n          customServiceConfig: |\n            [DEFAULT]\n            enabled_backends = lvm2\n          customServiceConfigSecrets:\n            - cinder-conf\n</code></pre>"},{"location":"openstack/cinder_adoption/#configuration-generation-helper-tool","title":"Configuration generation helper tool","text":"<p>Creating the right Cinder configuration files to deploy using Operators may sometimes be a complicated experience, especially the first times, so we have a helper tool that can create a draft of the files from a <code>cinder.conf</code> file.</p> <p>This tool is not meant to be a automation tool, it's mostly to help us get the gist of it, maybe point out some potential pitfalls and reminders.</p> <p>Attention: The tools requires <code>PyYAML</code> Python package to be installed (<code>pip install PyYAML</code>).</p> <p>This cinder-cfg.py script defaults to reading the <code>cinder.conf</code> file from the current directory (unless <code>--config</code> option is used) and outputs files to the current directory (unless <code>--out-dir</code> option is used).</p> <p>In the output directory we'll always get a <code>cinder.patch</code> file with the Cinder specific configuration patch to apply to the <code>OpenStackControlPlane</code> CR but we may also get an additional file called <code>cinder-prereq.yaml</code> file with some <code>Secrets</code> and <code>MachineConfigs</code>.</p> <p>Example of an invocation setting input and output explicitly to the defaults for a Ceph backend:</p> <pre><code>$ python cinder-cfg.py --config cinder.conf --out-dir ./\nWARNING:root:Cinder is configured to use ['/etc/cinder/policy.yaml'] as policy file, please ensure this file is available for the podified cinder services using \"extraMounts\" or remove the option.\n\nWARNING:root:Deployment uses Ceph, so make sure the Ceph credentials and configuration are present in OpenShift as a asecret and then use the extra volumes to make them available in all the services that would need them.\n\nWARNING:root:You were using user ['nova'] to talk to Nova, but in podified we prefer using the service keystone username, in this case ['cinder']. Dropping that configuration.\n\nWARNING:root:ALWAYS REVIEW RESULTS, OUTPUT IS JUST A ROUGH DRAFT!!\n\nOutput written at ./: cinder.patch\n</code></pre> <p>The script outputs some warnings to let us know things we may need to do manually -adding the custom policy, provide the ceph configuration files- and also let us know a change in how the <code>service_user</code> has been removed.</p> <p>A different example when using multiple backends, one of them being a 3PAR FC could be:</p> <pre><code>$ python cinder-cfg.py --config cinder.conf --out-dir ./\nWARNING:root:Cinder is configured to use ['/etc/cinder/policy.yaml'] as policy file, please ensure this file is available for the podified cinder services using \"extraMounts\" or remove the option.\n\nERROR:root:Backend hpe_fc requires a vendor container image, but there is no certified image available yet. Patch will use the last known image for reference, but IT WILL NOT WORK\n\nWARNING:root:Deployment uses Ceph, so make sure the Ceph credentials and configuration are present in OpenShift as a asecret and then use the extra volumes to make them available in all the services that would need them.\n\nWARNING:root:You were using user ['nova'] to talk to Nova, but in podified we prefer using the service keystone username, in this case ['cinder']. Dropping that configuration.\n\nWARNING:root:Configuration is using FC, please ensure all your OpenShift nodes have HBAs or use labels to ensure that Volume and Backup services are scheduled on nodes with HBAs.\n\nWARNING:root:ALWAYS REVIEW RESULTS, OUTPUT IS JUST A ROUGH DRAFT!!\n\nOutput written at ./: cinder.patch, cinder-prereq.yaml\n</code></pre> <p>In this case we can see that there are additional messages, so let's quickly go over them:</p> <ul> <li>There's one message mentioning how this backend driver needs external vendor dependencies so the standard container image will not work. Unfortunately this image is still not available, so an older image is used in the output patch file for reference. We can then replace this image with one we build ourselves or with a Red Hat official one once the image is available. In this case we can see in our <code>cinder.patch</code> file:   <pre><code>      cinderVolumes:\n      hpe-fc:\n        containerImage: registry.connect.redhat.com/hpe3parcinder/openstack-cinder-volume-hpe3parcinder17-0\n</code></pre></li> <li> <p>The FC message reminds us that this transport protocol requires specific HBA cards to be present on the nodes where cinder services are running.</p> </li> <li> <p>In this case we also see that it has created the <code>cinder-prereq.yaml</code> file and if we look into it we'll see there is one <code>MachineConfig</code> and one <code>Secret</code>. The <code>MachineConfig</code> is called <code>99-master-cinder-enable-multipathd</code> and like the name suggests enables multipathing on all the OCP worker nodes. The <code>Secret</code> is called <code>openstackcinder-volumes-hpe_fc</code> and contains the 3PAR backend configuration because it has sensitive information (credentials), and in the <code>cinder.patch</code> file we'll see that it uses this configuration:   <pre><code>   cinderVolumes:\n      hpe-fc:\n        customServiceConfigSecrets:\n        - openstackcinder-volumes-hpe_fc\n</code></pre></p> </li> </ul>"},{"location":"openstack/cinder_adoption/#procedure-cinder-adoption","title":"Procedure - Cinder adoption","text":"<p>Assuming we have already stopped cinder services, prepared the OpenShift nodes, deployed the OpenStack operators and a bare OpenStack manifest, and migrated the database, and prepared the patch manifest with the Cinder service configuration, all that's left is to apply the patch and wait for the operator to apply the changes and deploy the Cinder services.</p> <p>Our recommendation is to write the patch manifest into a file, for example <code>cinder.patch</code> and then apply it with something like:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch-file=cinder.patch\n</code></pre> <p>For example, for the RBD deployment from the Development Guide the <code>cinder.patch</code> would look like this:</p> <pre><code>spec:\n  extraMounts:\n  - extraVol:\n    - extraVolType: Ceph\n      mounts:\n      - mountPath: /etc/ceph\n        name: ceph\n        readOnly: true\n      propagation:\n      - CinderVolume\n      - CinderBackup\n      - Glance\n      volumes:\n      - name: ceph\n        projected:\n          sources:\n          - secret:\n              name: ceph-conf-files\n  cinder:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      secret: osp-secret\n      cinderAPI:\n        override:\n          service:\n            internal:\n              metadata:\n                annotations:\n                  metallb.universe.tf/address-pool: internalapi\n                  metallb.universe.tf/allow-shared-ip: internalapi\n                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n              spec:\n                type: LoadBalancer\n        replicas: 1\n        customServiceConfig: |\n          [DEFAULT]\n          default_volume_type=tripleo\n      cinderScheduler:\n        replicas: 1\n      cinderBackup:\n        networkAttachments:\n        - storage\n        replicas: 1\n        customServiceConfig: |\n          [DEFAULT]\n          backup_driver=cinder.backup.drivers.ceph.CephBackupDriver\n          backup_ceph_conf=/etc/ceph/ceph.conf\n          backup_ceph_user=openstack\n          backup_ceph_pool=backups\n      cinderVolumes:\n        ceph:\n          networkAttachments:\n          - storage\n          replicas: 1\n          customServiceConfig: |\n            [tripleo_ceph]\n            backend_host=hostgroup\n            volume_backend_name=tripleo_ceph\n            volume_driver=cinder.volume.drivers.rbd.RBDDriver\n            rbd_ceph_conf=/etc/ceph/ceph.conf\n            rbd_user=openstack\n            rbd_pool=volumes\n            rbd_flatten_volume_from_snapshot=False\n            report_discard_supported=True\n</code></pre> <p>Once the services have been deployed we'll need to clean up the old scheduler and backup services which will appear as being down while we have others that appear as being up:</p> <pre><code>openstack volume service list\n\n+------------------+------------------------+------+---------+-------+----------------------------+\n| Binary           | Host                   | Zone | Status  | State | Updated At                 |\n+------------------+------------------------+------+---------+-------+----------------------------+\n| cinder-backup    | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:59.000000 |\n| cinder-scheduler | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:29.000000 |\n| cinder-volume    | hostgroup@tripleo_ceph | nova | enabled | up    | 2023-06-28T17:00:03.000000 |\n| cinder-scheduler | cinder-scheduler-0     | nova | enabled | up    | 2023-06-28T17:00:02.000000 |\n| cinder-backup    | cinder-backup-0        | nova | enabled | up    | 2023-06-28T17:00:01.000000 |\n+------------------+------------------------+------+---------+-------+----------------------------+\n</code></pre> <p>In this case we need to remove services for hosts <code>standalone.localdomain</code></p> <pre><code>oc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-backup standalone.localdomain\noc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-scheduler standalone.localdomain\n</code></pre> <p>The reason why we haven't preserved the name of the backup service is because we have taken the opportunity to change its configuration to support Active-Active, even though we are not doing so right now because we have 1 replica.</p> <p>Now that we have the Cinder services running we know that the DB schema migration has been completed and we can proceed to apply the DB data migrations. While it is not necessary to run these data migrations at this precise moment, because we can just run them right before the next upgrade, we consider that for adoption it's best to run them now to make sure there are no issues before running production workloads on the deployment.</p> <p>The command to run the DB data migrations is:</p> <pre><code>oc exec -it cinder-scheduler-0 -- cinder-manage db online_data_migrations\n</code></pre>"},{"location":"openstack/cinder_adoption/#post-checks","title":"Post-checks","text":"<p>Before we can run any checks we need to set the right cloud configuration for the <code>openstack</code> command to be able to connect to our OpenShift control plane.</p> <p>Just like we did in the KeyStone adoption step we ensure we have the <code>openstack</code> alias defined:</p> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n</code></pre> <p>Now we can run a set of tests to confirm that the deployment is there using our old database contents:</p> <ul> <li>See that Cinder endpoints are defined and pointing to the podified   FQDNs:</li> </ul> <pre><code>openstack endpoint list --service cinderv3\n</code></pre> <ul> <li>Check that the cinder services are running and up. The API won't show but if   you get a response you know it's up as well:</li> </ul> <pre><code>openstack volume service list\n</code></pre> <ul> <li>Check that our old volume types, volumes, snapshots, and backups are there:</li> </ul> <pre><code>openstack volume type list\nopenstack volume list\nopenstack volume snapshot list\nopenstack volume backup list\n</code></pre> <p>To confirm that everything not only looks good but it's also properly working we recommend doing some basic operations:</p> <ul> <li> <p>Create a volume from an image to check that the connection to glance is   working.   <pre><code>openstack volume create --image cirros --bootable --size 1 disk_new\n</code></pre></p> </li> <li> <p>Backup the old attached volume to a new backup. Example:   <pre><code>openstack --os-volume-api-version 3.47 volume create --backup backup restored\n</code></pre></p> </li> </ul> <p>We don't boot a nova instance using the new volume from image or try to detach the old volume because nova and cinder are still not connected.</p>"},{"location":"openstack/edpm_adoption/","title":"EDPM adoption","text":""},{"location":"openstack/edpm_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed.</li> </ul>"},{"location":"openstack/edpm_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/edpm_adoption/#pre-checks","title":"Pre-checks","text":"<ul> <li>Make sure the IPAM is configured</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: network.openstack.org/v1beta1\nkind: NetConfig\nmetadata:\n  name: netconfig\nspec:\n  networks:\n  - name: CtlPlane\n    dnsDomain: ctlplane.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 192.168.122.120\n        start: 192.168.122.100\n      - end: 192.168.122.200\n        start: 192.168.122.150\n      cidr: 192.168.122.0/24\n      gateway: 192.168.122.1\n  - name: InternalApi\n    dnsDomain: internalapi.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.17.0.250\n        start: 172.17.0.100\n      cidr: 172.17.0.0/24\n      vlan: 20\n  - name: External\n    dnsDomain: external.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 10.0.0.250\n        start: 10.0.0.100\n      cidr: 10.0.0.0/24\n      gateway: 10.0.0.1\n  - name: Storage\n    dnsDomain: storage.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.18.0.250\n        start: 172.18.0.100\n      cidr: 172.18.0.0/24\n      vlan: 21\n  - name: StorageMgmt\n    dnsDomain: storagemgmt.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.20.0.250\n        start: 172.20.0.100\n      cidr: 172.20.0.0/24\n      vlan: 23\n  - name: Tenant\n    dnsDomain: tenant.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.19.0.250\n        start: 172.19.0.100\n      cidr: 172.19.0.0/24\n      vlan: 22\nEOF\n</code></pre>"},{"location":"openstack/edpm_adoption/#procedure-edpm-adoption","title":"Procedure - EDPM adoption","text":"<ul> <li>Create a ssh authentication secret for the EDPM nodes:</li> </ul> <p><pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\nkind: Secret\nmetadata:\n    name: dataplane-adoption-secret\n    namespace: openstack\ndata:\n    ssh-privatekey: |\n$(cat ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa | base64 | sed 's/^/        /')\nEOF\n</code></pre> * Create the Nova Metadata secret (Workaround while nova isn't adopted yet):</p> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\nkind: Secret\nmetadata:\n    name: nova-metadata-neutron-config\ndata:\n    05-nova-metadata.conf: |\n$(echo \"[DEFAULT]\\nnova_metadata_host = 1.2.3.4\\nnova_metadata_port = 8775\\nnova_metadata_protocol = http\\nmetadata_proxy_shared_secret = 1234567842\\n\" | base64 | sed 's/^/        /')\nEOF\n</code></pre> <ul> <li>Stop the nova services.</li> </ul> <pre><code># Update the services list to be stopped\n\nServicesToStop=(\"tripleo_nova_api_cron.service\"\n                \"tripleo_nova_api.service\"\n                \"tripleo_nova_compute.service\"\n                \"tripleo_nova_conductor.service\"\n                \"tripleo_nova_libvirt.target\"\n                \"tripleo_nova_metadata.service\"\n                \"tripleo_nova_migration_target.service\"\n                \"tripleo_nova_scheduler.service\"\n                \"tripleo_nova_virtlogd_wrapper.service\"\n                \"tripleo_nova_virtnodedevd.service\"\n                \"tripleo_nova_virtproxyd.service\"\n                \"tripleo_nova_virtqemud.service\"\n                \"tripleo_nova_virtsecretd.service\"\n                \"tripleo_nova_virtstoraged.service\"\n                \"tripleo_nova_vnc_proxy.service\")\n\necho \"Stopping nova services\"\n\nfor service in ${ServicesToStop[*]}; do\n    echo \"Stopping the $service in each controller node\"\n    $CONTROLLER1_SSH sudo systemctl stop $service\n    $CONTROLLER2_SSH sudo systemctl stop $service\n    $CONTROLLER3_SSH sudo systemctl stop $service\ndone\n</code></pre> <ul> <li>Deploy OpenStackDataPlaneNodeSet:</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: dataplane.openstack.org/v1beta1\nkind: OpenStackDataPlaneNodeSet\nmetadata:\n  name: openstack\nspec:\n  networkAttachments:\n      - ctlplane\n  preProvisioned: true\n  services:\n    - download-cache\n    - configure-network\n    - validate-network\n    - install-os\n    - configure-os\n    - run-os\n    - ovn\n  env:\n    - name: ANSIBLE_CALLBACKS_ENABLED\n      value: \"profile_tasks\"\n    - name: ANSIBLE_FORCE_COLOR\n      value: \"True\"\n    - name: ANSIBLE_ENABLE_TASK_DEBUGGER\n      value: \"True\"\n  nodes:\n    standalone:\n      hostName: standalone\n      ansible:\n        ansibleHost: 192.168.122.100\n      networks:\n      - defaultRoute: true\n        fixedIP: 192.168.122.100\n        name: CtlPlane\n        subnetName: subnet1\n      - name: InternalApi\n        subnetName: subnet1\n      - name: Storage\n        subnetName: subnet1\n      - name: Tenant\n        subnetName: subnet1\n  nodeTemplate:\n    ansibleSSHPrivateKeySecret: dataplane-adoption-secret\n    managementNetwork: ctlplane\n    ansible:\n      ansibleUser: root\n      ansiblePort: 22\n      ansibleVars:\n        service_net_map:\n          nova_api_network: internal_api\n          nova_libvirt_network: internal_api\n\n        # edpm_network_config\n        # Default nic config template for a EDPM compute node\n        # These vars are edpm_network_config role vars\n        edpm_network_config_override: \"\"\n        edpm_network_config_template: |\n           ---\n           {% set mtu_list = [ctlplane_mtu] %}\n           {% for network in role_networks %}\n           {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}\n           {%- endfor %}\n           {% set min_viable_mtu = mtu_list | max %}\n           network_config:\n           - type: ovs_bridge\n             name: {{ neutron_physical_bridge_name }}\n             mtu: {{ min_viable_mtu }}\n             use_dhcp: false\n             dns_servers: {{ ctlplane_dns_nameservers }}\n             domain: {{ dns_search_domains }}\n             addresses:\n             - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}\n             routes: {{ ctlplane_host_routes }}\n             members:\n             - type: interface\n               name: nic1\n               mtu: {{ min_viable_mtu }}\n               # force the MAC address of the bridge to this interface\n               primary: true\n           {% for network in role_networks %}\n             - type: vlan\n               mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}\n               vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}\n               addresses:\n               - ip_netmask:\n                   {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}\n               routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}\n           {% endfor %}\n\n        edpm_network_config_hide_sensitive_logs: false\n        #\n        # These vars are for the network config templates themselves and are\n        # considered EDPM network defaults.\n        neutron_physical_bridge_name: br-ctlplane\n        neutron_public_interface_name: eth0\n        role_networks:\n        - InternalApi\n        - Storage\n        - Tenant\n        networks_lower:\n          External: external\n          InternalApi: internal_api\n          Storage: storage\n          Tenant: tenant\n\n        # edpm_nodes_validation\n        edpm_nodes_validation_validate_controllers_icmp: false\n        edpm_nodes_validation_validate_gateway_icmp: false\n\n        edpm_chrony_ntp_servers:\n        - clock.redhat.com\n        - clock2.redhat.com\n\n        edpm_ovn_controller_agent_image: quay.io/podified-antelope-centos9/openstack-ovn-controller:current-podified\n        edpm_iscsid_image: quay.io/podified-antelope-centos9/openstack-iscsid:current-podified\n        edpm_logrotate_crond_image: quay.io/podified-antelope-centos9/openstack-cron:current-podified\n        edpm_nova_compute_container_image: quay.io/podified-antelope-centos9/openstack-nova-compute:current-podified\n        edpm_nova_libvirt_container_image: quay.io/podified-antelope-centos9/openstack-nova-libvirt:current-podified\n        edpm_ovn_metadata_agent_image: quay.io/podified-antelope-centos9/openstack-neutron-metadata-agent-ovn:current-podified\n\n        gather_facts: false\n        enable_debug: false\n        # edpm firewall, change the allowed CIDR if needed\n        edpm_sshd_configure_firewall: true\n        edpm_sshd_allowed_ranges: ['192.168.122.0/24']\n        # SELinux module\n        edpm_selinux_mode: enforcing\n        plan: overcloud\nEOF\n</code></pre> <ul> <li>Deploy OpenStackDataPlaneDeployment:</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: dataplane.openstack.org/v1beta1\nkind: OpenStackDataPlaneDeployment\nmetadata:\n  name: openstack\nspec:\n  nodeSets:\n  - openstack\nEOF\n</code></pre>"},{"location":"openstack/edpm_adoption/#post-checks","title":"Post-checks","text":"<ul> <li> <p>Check if all the Ansible EE pods reaches <code>Completed</code> status:</p> <p><pre><code># watching the pods\nwatch oc get pod -l app=openstackansibleee\n</code></pre> <pre><code># following the ansible logs with:\noc logs -l app=openstackansibleee -f --max-log-requests 10\n</code></pre></p> </li> <li> <p>Wait for the dataplane node set to reach the Ready status:     <pre><code>oc wait --for condition=Ready osdpns/openstack --timeout=30m\n</code></pre></p> </li> </ul>"},{"location":"openstack/glance_adoption/","title":"Glance adoption","text":"<p>Adopting Glance means that an existing <code>OpenStackControlPlane</code> CR, where Glance is supposed to be disabled, should be patched to start the service with the configuration parameters provided by the source environment.</p> <p>When the procedure is over, the expectation is to see the <code>GlanceAPI</code> service up and running: the <code>Keystone endpoints</code> should be updated and the same backend of the source Cloud will be available. If the conditions above are met, the adoption is considered concluded.</p> <p>This guide also assumes that:</p> <ol> <li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li> <li>A <code>SNO</code> / <code>CodeReadyContainers</code> is running on the other side;</li> <li>(optional) an internal/external <code>Ceph</code> cluster is reachable by both <code>crc</code> and <code>TripleO</code></li> </ol>"},{"location":"openstack/glance_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, MariaDB and Keystone   should be already adopted.</li> </ul>"},{"location":"openstack/glance_adoption/#procedure-glance-adoption","title":"Procedure - Glance adoption","text":"<p>As already done for Keystone, the Glance Adoption follows the same pattern.</p>"},{"location":"openstack/glance_adoption/#using-local-storage-backend","title":"Using local storage backend","text":"<p>When Glance should be deployed with local storage backend (not Ceph), patch OpenStackControlPlane to deploy Glance:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  glance:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      storageClass: \"local-storage\"\n      storageRequest: 10G\n      glanceAPIInternal:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n        networkAttachments:\n        - storage\n      glanceAPIExternal:\n        networkAttachments:\n        - storage\n'\n</code></pre>"},{"location":"openstack/glance_adoption/#using-ceph-storage-backend","title":"Using Ceph storage backend","text":"<p>If a Ceph backend is used, the <code>customServiceConfig</code> parameter should be used to inject the right configuration to the <code>GlanceAPI</code> instance.</p> <p>Make sure the Ceph-related secret (<code>ceph-conf-files</code>) was created in the <code>openstack</code> namespace and that the <code>extraMounts</code> property of the <code>OpenStackControlPlane</code> CR has been configured properly. These tasks are described in an earlier Adoption step Ceph storage backend configuration.</p> <pre><code>cat &lt;&lt; EOF &gt; glance_patch.yaml\nspec:\n  glance:\n    enabled: true\n    template:\n      databaseInstance: openstack\n      customServiceConfig: |\n        [DEFAULT]\n        enabled_backends=default_backend:rbd\n        [glance_store]\n        default_backend=default_backend\n        [default_backend]\n        rbd_store_ceph_conf=/etc/ceph/ceph.conf\n        rbd_store_user=openstack\n        rbd_store_pool=images\n        store_description=Ceph glance store backend.\n      storageClass: \"local-storage\"\n      storageRequest: 10G\n      glanceAPIInternal:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n        networkAttachments:\n        - storage\n      glanceAPIExternal:\n        networkAttachments:\n        - storage\nEOF\n</code></pre> <p>If you have previously backup your Openstack services configuration file from the old environment: pull openstack configuration os-diff you can use os-diff to compare and make sure the configuration is correct.</p> <pre><code>pushd os-diff\n./os-diff cdiff --service glance -c /tmp/collect_tripleo_configs/glance/etc/glance/glance-api.conf -o glance_patch.yaml\n</code></pre> <p>This will producre the difference between both ini configuration files.</p> <p>Patch OpenStackControlPlane to deploy Glance with Ceph backend:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch-file glance_patch.yaml\n</code></pre>"},{"location":"openstack/glance_adoption/#post-checks","title":"Post-checks","text":""},{"location":"openstack/glance_adoption/#test-the-glance-service-from-the-openstack-cli","title":"Test the glance service from the OpenStack CLI","text":"<p>You can compare and make sure the configuration has been correctly applied to the glance pods by running</p> <pre><code>./os-diff cdiff --service glance -c /etc/glance/glance.conf.d/02-config.conf  -o glance_patch.yaml --frompod -p glance-api\n</code></pre> <p>If no line appear, then the configuration is correctly done.</p> <p>Inspect the resulting glance pods:</p> <pre><code>GLANCE_POD=`oc get pod |grep glance-external-api | cut -f 1 -d' '`\noc exec -t $GLANCE_POD -c glance-api -- cat /etc/glance/glance.conf.d/02-config.conf\n\n[DEFAULT]\nenabled_backends=default_backend:rbd\n[glance_store]\ndefault_backend=default_backend\n[default_backend]\nrbd_store_ceph_conf=/etc/ceph/ceph.conf\nrbd_store_user=openstack\nrbd_store_pool=images\nstore_description=Ceph glance store backend.\n\noc exec -t $GLANCE_POD -c glance-api -- ls /etc/ceph\nceph.client.openstack.keyring\nceph.conf\n</code></pre> <p>Ceph secrets are properly mounted, at this point let's move to the OpenStack CLI and check the service is active and the endpoints are properly updated.</p> <pre><code>(openstack)$ service list | grep image\n\n| fc52dbffef36434d906eeb99adfc6186 | glance    | image        |\n\n(openstack)$ endpoint list | grep image\n\n| 569ed81064f84d4a91e0d2d807e4c1f1 | regionOne | glance       | image        | True    | internal  | http://glance-internal-openstack.apps-crc.testing   |\n| 5843fae70cba4e73b29d4aff3e8b616c | regionOne | glance       | image        | True    | public    | http://glance-public-openstack.apps-crc.testing     |\n| 709859219bc24ab9ac548eab74ad4dd5 | regionOne | glance       | image        | True    | admin     | http://glance-admin-openstack.apps-crc.testing      |\n</code></pre> <p>Check the images that we previously listed in the source Cloud are available in the adopted service:</p> <pre><code>(openstack)$ image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| c3158cad-d50b-452f-bec1-f250562f5c1f | cirros | active |\n+--------------------------------------+--------+--------+\n</code></pre>"},{"location":"openstack/glance_adoption/#image-upload","title":"Image upload","text":"<p>We can test that an image can be created on from the adopted service.</p> <pre><code>(openstack)$ alias openstack=\"oc exec -t openstackclient -- openstack\"\n(openstack)$ curl -L -o /tmp/cirros-0.5.2-x86_64-disk.img http://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img\n    qemu-img convert -O raw /tmp/cirros-0.5.2-x86_64-disk.img /tmp/cirros-0.5.2-x86_64-disk.img.raw\n    openstack image create --container-format bare --disk-format raw --file /tmp/cirros-0.5.2-x86_64-disk.img.raw cirros2\n    openstack image list\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n100   273  100   273    0     0   1525      0 --:--:-- --:--:-- --:--:--  1533\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n100 15.5M  100 15.5M    0     0  17.4M      0 --:--:-- --:--:-- --:--:-- 17.4M\n\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n| Field            | Value                                                                                                                                      |\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n| container_format | bare                                                                                                                                       |\n| created_at       | 2023-01-31T21:12:56Z                                                                                                                       |\n| disk_format      | raw                                                                                                                                        |\n| file             | /v2/images/46a3eac1-7224-40bc-9083-f2f0cd122ba4/file                                                                                       |\n| id               | 46a3eac1-7224-40bc-9083-f2f0cd122ba4                                                                                                       |\n| min_disk         | 0                                                                                                                                          |\n| min_ram          | 0                                                                                                                                          |\n| name             | cirros                                                                                                                                     |\n| owner            | 9f7e8fdc50f34b658cfaee9c48e5e12d                                                                                                           |\n| properties       | os_hidden='False', owner_specified.openstack.md5='', owner_specified.openstack.object='images/cirros', owner_specified.openstack.sha256='' |\n| protected        | False                                                                                                                                      |\n| schema           | /v2/schemas/image                                                                                                                          |\n| status           | queued                                                                                                                                     |\n| tags             |                                                                                                                                            |\n| updated_at       | 2023-01-31T21:12:56Z                                                                                                                       |\n| visibility       | shared                                                                                                                                     |\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 46a3eac1-7224-40bc-9083-f2f0cd122ba4 | cirros2| active |\n| c3158cad-d50b-452f-bec1-f250562f5c1f | cirros | active |\n+--------------------------------------+--------+--------+\n\n\n(openstack)$ oc rsh ceph\nsh-4.4$ ceph -s\nr  cluster:\n    id:     432d9a34-9cee-4109-b705-0c59e8973983\n    health: HEALTH_OK\n\n  services:\n    mon: 1 daemons, quorum a (age 4h)\n    mgr: a(active, since 4h)\n    osd: 1 osds: 1 up (since 4h), 1 in (since 4h)\n\n  data:\n    pools:   5 pools, 160 pgs\n    objects: 46 objects, 224 MiB\n    usage:   247 MiB used, 6.8 GiB / 7.0 GiB avail\n    pgs:     160 active+clean\n\nsh-4.4$ rbd -p images ls\n46a3eac1-7224-40bc-9083-f2f0cd122ba4\nc3158cad-d50b-452f-bec1-f250562f5c1f\n</code></pre>"},{"location":"openstack/horizon_adoption/","title":"Horizon adoption","text":""},{"location":"openstack/horizon_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, Memcached and   keystone should be already adopted.</li> </ul>"},{"location":"openstack/horizon_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/horizon_adoption/#procedure-horizon-adoption","title":"Procedure - Horizon adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Horizon:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  horizon:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      memcachedInstance: memcached\n      secret: osp-secret\n'\n</code></pre>"},{"location":"openstack/horizon_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Horizon instance is successfully deployed and ready</li> </ul> <pre><code>oc get horizon\n</code></pre> <ul> <li>Check that dashboard is reachable and returns status code <code>200</code></li> </ul> <pre><code>PUBLIC_URL=$(oc get horizon horizon -o jsonpath='{.status.endpoint}')\ncurl --silent --output /dev/stderr --head --write-out \"%{http_code}\" \"$PUBLIC_URL/dashboard/auth/login/?next=/dashboard/\" | grep 200\n</code></pre>"},{"location":"openstack/ironic_adoption/","title":"Ironic adoption","text":""},{"location":"openstack/ironic_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/ironic_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/ironic_adoption/#pre-checks","title":"Pre-checks","text":"<p>TODO</p>"},{"location":"openstack/ironic_adoption/#procedure-ironic-adoption","title":"Procedure - Ironic adoption","text":"<p>TODO</p>"},{"location":"openstack/ironic_adoption/#post-checks","title":"Post-checks","text":"<p>TODO</p>"},{"location":"openstack/keystone_adoption/","title":"Keystone adoption","text":""},{"location":"openstack/keystone_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/keystone_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/keystone_adoption/#pre-checks","title":"Pre-checks","text":""},{"location":"openstack/keystone_adoption/#procedure-keystone-adoption","title":"Procedure - Keystone adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Keystone:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  keystone:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n      databaseInstance: openstack\n      secret: osp-secret\n'\n</code></pre> <ul> <li>Create alias to use <code>openstack</code> command in the adopted deployment:</li> </ul> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n</code></pre> <ul> <li>Clean up old services and endpoints that still point to the old   control plane (everything except Keystone service and endpoints):</li> </ul> <pre><code>openstack endpoint list | grep keystone | awk '/admin/{ print $2; }' | xargs ${BASH_ALIASES[openstack]} endpoint delete || true\n\nfor service in cinderv3 glance neutron nova placement swift; do\n  openstack service list | awk \"/ $service /{ print \\$2; }\" | xargs ${BASH_ALIASES[openstack]} service delete || true\ndone\n</code></pre>"},{"location":"openstack/keystone_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Keystone endpoints are defined and pointing to the podified   FQDNs:</li> </ul> <pre><code>openstack endpoint list | grep keystone\n</code></pre>"},{"location":"openstack/mariadb_copy/","title":"MariaDB data copy","text":"<p>This document describes how to move the databases from the original OpenStack deployment to the MariaDB instances in the OpenShift cluster.</p>"},{"location":"openstack/mariadb_copy/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>Make sure the previous Adoption steps have been performed successfully.</p> </li> <li> <p>The OpenStackControlPlane resource must be already created at this point.</p> </li> <li> <p>Podified MariaDB and RabbitMQ are running. No other podified     control plane services are running.</p> </li> <li> <p>OpenStack services have been stopped</p> </li> <li> <p>There must be network routability between:</p> <ul> <li> <p>The adoption host and the original MariaDB.</p> </li> <li> <p>The adoption host and the podified MariaDB.</p> </li> <li> <p>Note that this routability requirement may change in the   future, e.g. we may require routability from the original MariaDB to   podified MariaDB.</p> </li> </ul> </li> <li> <p>Podman package is installed</p> </li> <li> <p><code>CONTROLLER1_SSH</code>, <code>CONTROLLER2_SSH</code>, and <code>CONTROLLER3_SSH</code> are configured.</p> </li> </ul>"},{"location":"openstack/mariadb_copy/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified\n\nPODIFIED_MARIADB_IP=$(oc get svc --selector \"cr=mariadb-openstack\" -ojsonpath='{.items[0].spec.clusterIP}')\nPODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector \"cr=mariadb-openstack-cell1\" -ojsonpath='{.items[0].spec.clusterIP}')\nPODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)\n\n# Replace with your environment's MariaDB IP:\nSOURCE_MARIADB_IP=192.168.122.100\nSOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }')\n\n# The CHARACTER_SET and collation should match the source DB\n# if the do not then it will break foreign key relationships\n# for any tables that are created in the future as part of db sync\nCHARACTER_SET=utf8\nCOLLATION=utf8_general_ci\n</code></pre>"},{"location":"openstack/mariadb_copy/#pre-checks","title":"Pre-checks","text":"<ul> <li>Test connection to the original DB (show databases):</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \\\n    mysql -h \"$SOURCE_MARIADB_IP\" -uroot \"-p$SOURCE_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\n</code></pre> <ul> <li>Run mysqlcheck on the original DB to look for things that are not OK:</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \\\n    mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root \"-p$SOURCE_DB_ROOT_PASSWORD\" | grep -v OK\n</code></pre> <ul> <li>Test connection to podified DBs (show databases):</li> </ul> <pre><code>oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\n    mysql -h \"$PODIFIED_MARIADB_IP\" -uroot \"-p$PODIFIED_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\n    mysql -h \"$PODIFIED_CELL1_MARIADB_IP\" -uroot \"-p$PODIFIED_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\n</code></pre>"},{"location":"openstack/mariadb_copy/#procedure-data-copy","title":"Procedure - data copy","text":"<ul> <li>Create a temporary folder to store DB dumps and make sure it's the   working directory for the following steps:</li> </ul> <pre><code>mkdir ~/adoption-db\ncd ~/adoption-db\n</code></pre> <ul> <li>Create a dump of the original databases:</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash &lt;&lt;EOF\n\n# Note we do not want to dump the information and performance schema tables so we filter them\nmysql -h ${SOURCE_MARIADB_IP} -u root \"-p${SOURCE_DB_ROOT_PASSWORD}\" -N -e 'show databases' | grep -E -v 'schema|mysql' | while read dbname; do\n    echo \"Dumping \\${dbname}\"\n    mysqldump -h $SOURCE_MARIADB_IP -uroot \"-p$SOURCE_DB_ROOT_PASSWORD\" \\\n        --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \\\n        \"\\${dbname}\" &gt; \"\\${dbname}\".sql\ndone\n\nEOF\n</code></pre> <ul> <li>Restore the databases from .sql files into the podified MariaDB:</li> </ul> <pre><code># db schemas to rename on import\ndeclare -A db_name_map\ndb_name_map[\"nova\"]=\"nova_cell1\"\ndb_name_map[\"ovs_neutron\"]=\"neutron\"\n\n# db servers to import into\ndeclare -A db_server_map\ndb_server_map[\"default\"]=${PODIFIED_MARIADB_IP}\ndb_server_map[\"nova_cell1\"]=${PODIFIED_CELL1_MARIADB_IP}\n\n# db server root password map\ndeclare -A db_server_password_map\ndb_server_password_map[\"default\"]=${PODIFIED_DB_ROOT_PASSWORD}\ndb_server_password_map[\"nova_cell1\"]=${PODIFIED_DB_ROOT_PASSWORD}\n\nall_db_files=$(ls *.sql)\nfor db_file in ${all_db_files}; do\n    db_name=$(echo ${db_file} | awk -F'.' '{ print $1; }')\n    if [[ -v \"db_name_map[${db_name}]\" ]]; then\n        echo \"renaming ${db_name} to ${db_name_map[${db_name}]}\"\n        db_name=${db_name_map[${db_name}]}\n    fi\n    db_server=${db_server_map[\"default\"]}\n    if [[ -v \"db_server_map[${db_name}]\" ]]; then\n        db_server=${db_server_map[${db_name}]}\n    fi\n    db_password=${db_server_password_map[\"default\"]}\n    if [[ -v \"db_server_password_map[${db_name}]\" ]]; then\n        db_password=${db_server_password_map[${db_name}]}\n    fi\n    echo \"creating ${db_name} in ${db_server}\"\n    container_name=$(echo \"mariadb-client-${db_name}-create\" | sed 's/_/-/g')\n    oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \\\n        mysql -h \"${db_server}\" -uroot \"-p${db_password}\" &lt;&lt; EOF\nCREATE DATABASE IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};\nEOF\n    echo \"importing ${db_name} into ${db_server}\"\n    container_name=$(echo \"mariadb-client-${db_name}-restore\" | sed 's/_/-/g')\n    oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \\\n        mysql -h \"${db_server}\" -uroot \"-p${db_password}\" \"${db_name}\" &lt; \"${db_file}\"\ndone\n</code></pre>"},{"location":"openstack/mariadb_copy/#post-checks","title":"Post-checks","text":"<ul> <li>Check that the databases were imported correctly:</li> </ul> <pre><code>oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep keystone\n# ensure neutron db is renamed from ovs_neutron\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep neutron\n# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_CELL1_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep nova_cell1\n</code></pre> <ul> <li>During the pre/post checks the pod <code>mariadb-client</code> might have returned a pod security warning   related to the <code>restricted:latest</code> security context constraint. This is due to default security   context constraints and will not prevent pod creation by the admission controller. You'll see a   warning for the short-lived pod but it will not interfere with functionality.   For more info visit here</li> </ul>"},{"location":"openstack/neutron_adoption/","title":"Neutron adoption","text":"<p>Adopting Neutron means that an existing <code>OpenStackControlPlane</code> CR, where Neutron is supposed to be disabled, should be patched to start the service with the configuration parameters provided by the source environment.</p> <p>When the procedure is over, the expectation is to see the <code>NeutronAPI</code> service up and running: the <code>Keystone endpoints</code> should be updated and the same backend of the source Cloud will be available. If the conditions above are met, the adoption is considered concluded.</p> <p>This guide also assumes that:</p> <ol> <li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li> <li>A <code>SNO</code> / <code>CodeReadyContainers</code> is running on the other side.</li> </ol>"},{"location":"openstack/neutron_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, MariaDB and Keystone   should be already adopted.</li> </ul>"},{"location":"openstack/neutron_adoption/#procedure-neutron-adoption","title":"Procedure - Neutron adoption","text":"<p>As already done for Keystone, the Neutron Adoption follows the same pattern.</p> <p>Patch OpenStackControlPlane to deploy Neutron:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  neutron:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n      databaseInstance: openstack\n      secret: osp-secret\n      networkAttachments:\n      - internalapi\n'\n</code></pre>"},{"location":"openstack/neutron_adoption/#post-checks","title":"Post-checks","text":""},{"location":"openstack/neutron_adoption/#inspect-the-resulting-neutron-pods","title":"Inspect the resulting neutron pods","text":"<pre><code>NEUTRON_API_POD=`oc get pods -l service=neutron | tail -n 1 | cut -f 1 -d' '`\noc exec -t $NEUTRON_API_POD -c neutron-api -- cat /etc/neutron/neutron.conf\n</code></pre>"},{"location":"openstack/neutron_adoption/#check-that-neutron-api-service-is-registered-in-keystone","title":"Check that Neutron API service is registered in Keystone","text":"<pre><code>openstack service list | grep network\n</code></pre> <pre><code>openstack endpoint list | grep network\n\n| 6a805bd6c9f54658ad2f24e5a0ae0ab6 | regionOne | neutron      | network      | True    | public    | http://neutron-public-openstack.apps-crc.testing  |\n| b943243e596847a9a317c8ce1800fa98 | regionOne | neutron      | network      | True    | internal  | http://neutron-internal.openstack.svc:9696        |\n| f97f2b8f7559476bb7a5eafe3d33cee7 | regionOne | neutron      | network      | True    | admin     | http://192.168.122.99:9696                        |\n</code></pre>"},{"location":"openstack/neutron_adoption/#create-sample-resources","title":"Create sample resources","text":"<p>We can now test that user can create networks, subnets, ports, routers etc.</p> <pre><code>openstack network create net\nopenstack subnet create --network net --subnet-range 10.0.0.0/24 subnet\nopenstack router create router\n</code></pre> <p>NOTE: this page should be expanded to include information on SR-IOV adoption.</p>"},{"location":"openstack/node-selector/","title":"Node Selector","text":"<p>There are a variety of reasons why we may want to restrict the nodes where OpenStack services can be placed:</p> <ul> <li>Hardware requirements: System memory, Disk space, Cores, HBAs</li> <li>Limit the impact of the OpenStack services on other OpenShift workloads.</li> <li>Avoid collocating OpenStack services.</li> </ul> <p>The mechanism provided by the OpenStack operators to achieve this is through the use of labels.</p> <p>We would either label the OpenShift nodes or use existing labels they already have, and then use those labels in the OpenStack manifests in the <code>nodeSelector</code> field.</p> <p>The <code>nodeSelector</code> field in the OpenStack manifests follows the standard OpenShift <code>nodeSelector</code> field, please refer to the OpenShift documentation on the matter additional information.</p> <p>This field is present at all the different levels of the OpenStack manifests:</p> <ul> <li>Deployment: The <code>OpenStackControlPlane</code> object.</li> <li>Component: For example the <code>cinder</code> element in the <code>OpenStackControlPlane</code>.</li> <li>Service: For example the <code>cinderVolume</code> element within the <code>cinder</code> element   in the <code>OpenStackControlPlane</code>.</li> </ul> <p>This allows a fine grained control of the placement of the OpenStack services with minimal repetition.</p> <p>Values of the <code>nodeSelector</code> are propagated to the next levels unless they are overwritten. This means that a <code>nodeSelector</code> value at the deployment level will affect all the OpenStack services.</p> <p>For example we can add label <code>type: openstack</code> to any 3 OpenShift nodes:</p> <pre><code>$ oc label nodes worker0 type=openstack\n$ oc label nodes worker1 type=openstack\n$ oc label nodes worker2 type=openstack\n</code></pre> <p>And then in our <code>OpenStackControlPlane</code> we can use the label to place all the services in those 3 nodes:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  nodeSelector:\n    type: openstack\n&lt; . . . &gt;\n</code></pre> <p>What if we don't mind where any OpenStack services go but the cinder volume and backup services because we are using FC and we only have HBAs on a subset of nodes? Then we can just use the selector on for those specific services, which for the sake of this example we'll assume they have the label <code>fc_card: true</code>:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  cinder:\n    template:\n      cinderVolumes:\n          pure_fc:\n            nodeSelector:\n              fc_card: true\n&lt; . . . &gt;\n          lvm-iscsi:\n            nodeSelector:\n              fc_card: true\n&lt; . . . &gt;\n      cinderBackup:\n          nodeSelector:\n            fc_card: true\n&lt; . . . &gt;\n</code></pre> <p>The Cinder operator does not currently have the possibility of defining the <code>nodeSelector</code> in <code>cinderVolumes</code>, so we need to specify it on each of the backends.</p> <p>It's possible to leverage labels added by the node feature discovery operator to place OpenStack services.</p>"},{"location":"openstack/node-selector/#machineconfig","title":"MachineConfig","text":"<p>Some services require us to have services or kernel modules running on the hosts where they run, for example <code>iscsid</code> or <code>multipathd</code> daemons, or the <code>nvme-fabrics</code> kernel module.</p> <p>For those cases we'll use <code>MachineConfig</code> manifests, and if we are restricting the nodes we are placing the OpenStack services using the <code>nodeSelector</code> then we'll also want to limit where the <code>MachineConfig</code> is applied.</p> <p>To define where the <code>MachineConfig</code> can be applied we'll need to use a <code>MachineConfigPool</code> that links the <code>MachineConfig</code> to the nodes.</p> <p>For example to be able to limit <code>MachineConfig</code> to the 3 OpenShift nodes we marked with the <code>type: openstack</code> label we would create the <code>MachineConfigPool</code> like this:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfigPool\nmetadata:\n  name: openstack\nspec:\n  machineConfigSelector:\n    matchLabels:\n      machineconfiguration.openshift.io/role: openstack\n  nodeSelector:\n    matchLabels:\n      type: openstack\n</code></pre> <p>And then we could use it in the <code>MachineConfig</code>:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: openstack\n&lt; . . . &gt;\n</code></pre> <p>Refer to the OpenShift documentation for additional information on <code>MachineConfig</code> and <code>MachineConfigPools</code></p> <p>WARNING: Applying a <code>MachineConfig</code> to an OpenShift node will make the node reboot.</p>"},{"location":"openstack/ovn_adoption/","title":"OVN data migration","text":"<p>This document describes how to move OVN northbound and southbound databases from the original OpenStack deployment to ovsdb-server instances running in the OpenShift cluster.</p>"},{"location":"openstack/ovn_adoption/#rationale","title":"Rationale","text":"<p>While it may be argued that the podified Neutron ML2/OVN driver and OVN northd service will reconstruct the databases on startup, the reconstruction may be time consuming on large existing clusters. The procedure below allows to speed up data migration and avoid unnecessary data plane disruptions due to incomplete OpenFlow table contents.</p>"},{"location":"openstack/ovn_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Make sure the previous Adoption steps have been performed successfully.</li> <li>The OpenStackControlPlane resource must be already created at this point.</li> <li>NetworkAttachmentDefinition CRDs for the original cluster are already     defined. Specifically, openstack/internalapi network is defined.</li> <li>Podified MariaDB and RabbitMQ may already run. Neutron and OVN are not     running yet.</li> <li>Original OVN is older or equal to the podified version.</li> <li>There must be network routability between:<ul> <li>The adoption host and the original OVN.</li> <li>The adoption host and the podified OVN.</li> </ul> </li> </ul>"},{"location":"openstack/ovn_adoption/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>OVSDB_IMAGE=quay.io/podified-antelope-centos9/openstack-ovn-base:current-podified\nSOURCE_OVSDB_IP=172.17.1.49\n\n# ssh commands to reach the original controller machines\nCONTROLLER_SSH=\"ssh -F ~/director_standalone/vagrant_ssh_config vagrant@standalone\"\n\n# ssh commands to reach the original compute machines\nCOMPUTE_SSH=\"ssh -F ~/director_standalone/vagrant_ssh_config vagrant@standalone\"\n</code></pre>"},{"location":"openstack/ovn_adoption/#procedure","title":"Procedure","text":"<ul> <li>Stop OVN northd on all original cluster controllers.</li> </ul> <pre><code>${CONTROLLER_SSH} sudo systemctl stop tripleo_ovn_cluster_northd.service\n</code></pre> <ul> <li>Backup OVN databases.</li> </ul> <pre><code>client=\"podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE ovsdb-client\"\n${client} backup tcp:$SOURCE_OVSDB_IP:6641 &gt; ovs-nb.db\n${client} backup tcp:$SOURCE_OVSDB_IP:6642 &gt; ovs-sb.db\n</code></pre> <ul> <li>Start podified OVN database services prior to import.</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  ovn:\n\u00a0 \u00a0 enabled: true\n    template:\n      ovnDBCluster:\n        ovndbcluster-nb:\n          dbType: NB\n          storageRequest: 10G\n          networkAttachment: internalapi\n        ovndbcluster-sb:\n          dbType: SB\n          storageRequest: 10G\n          networkAttachment: internalapi\n'\n</code></pre> <ul> <li>Fetch podified OVN IP addresses.</li> </ul> <pre><code>PODIFIED_OVSDB_NB_IP=$(kubectl get po ovsdbserver-nb-0 -o jsonpath='{.metadata.annotations.k8s\\.v1\\.cni\\.cncf\\.io/network-status}' | jq 'map(. | select(.name==\"openstack/internalapi\"))[0].ips[0]' | tr -d '\"')\nPODIFIED_OVSDB_SB_IP=$(kubectl get po ovsdbserver-sb-0 -o jsonpath='{.metadata.annotations.k8s\\.v1\\.cni\\.cncf\\.io/network-status}' | jq 'map(. | select(.name==\"openstack/internalapi\"))[0].ips[0]' | tr -d '\"')\n</code></pre> <ul> <li>Upgrade database schema for the backup files.</li> </ul> <pre><code>podman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client get-schema tcp:$PODIFIED_OVSDB_NB_IP:6641 &gt; ./ovs-nb.ovsschema &amp;&amp; ovsdb-tool convert ovs-nb.db ./ovs-nb.ovsschema\"\npodman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client get-schema tcp:$PODIFIED_OVSDB_SB_IP:6642 &gt; ./ovs-sb.ovsschema &amp;&amp; ovsdb-tool convert ovs-sb.db ./ovs-sb.ovsschema\"\n</code></pre> <ul> <li>Restore database backup to podified OVN database servers.</li> </ul> <pre><code>podman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client restore tcp:$PODIFIED_OVSDB_NB_IP:6641 &lt; ovs-nb.db\"\npodman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client restore tcp:$PODIFIED_OVSDB_SB_IP:6642 &lt; ovs-sb.db\"\n</code></pre> <ul> <li>Check that podified OVN databases contain objects from backup, e.g.:</li> </ul> <pre><code>oc exec -it ovsdbserver-nb-0 -- ovn-nbctl show\noc exec -it ovsdbserver-sb-0 -- ovn-sbctl list Chassis\n</code></pre> <ul> <li>Switch ovn-remote on compute nodes to point to the new podified database.</li> </ul> <pre><code>${COMPUTE_SSH} sudo podman exec -it ovn_controller ovs-vsctl set open . external_ids:ovn-remote=tcp:$PODIFIED_OVSDB_SB_IP:6642\n</code></pre> <p>You should now see the following warning in the <code>ovn_controller</code> container logs:</p> <pre><code>2023-03-16T21:40:35Z|03095|ovsdb_cs|WARN|tcp:172.17.1.50:6642: clustered database server has stale data; trying another server\n</code></pre> <ul> <li>Reset RAFT state for all compute ovn-controller instances.</li> </ul> <pre><code>${COMPUTE_SSH} sudo podman exec -it ovn_controller ovn-appctl -t ovn-controller sb-cluster-state-reset\n</code></pre> <p>This should complete connection of the controller process to the new remote. See in logs:</p> <pre><code>2023-03-16T21:42:31Z|03134|main|INFO|Resetting southbound database cluster state\n2023-03-16T21:42:33Z|03135|reconnect|INFO|tcp:172.17.1.50:6642: connected\n</code></pre> <ul> <li>Alternatively, just restart ovn-controller on original compute nodes.</li> </ul> <pre><code>$ ${COMPUTE_SSH} sudo systemctl restart tripleo_ovn_controller.service\n</code></pre> <ul> <li>Finally, you can start <code>ovn-northd</code> service that will keep OVN northbound and southbound databases in sync.</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  ovn:\n\u00a0 \u00a0 enabled: true\n    template:\n      ovnNorthd:\n        networkAttachment: internalapi\n'\n</code></pre>"},{"location":"openstack/placement_adoption/","title":"Placement adoption","text":""},{"location":"openstack/placement_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/placement_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/placement_adoption/#procedure-placement-adoption","title":"Procedure - Placement adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Placement:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  placement:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      secret: osp-secret\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n'\n</code></pre>"},{"location":"openstack/placement_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Placement endpoints are defined and pointing to the   podified FQDNs and that Placement API responds.</li> </ul> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\nopenstack endpoint list | grep placement\n\n\n# Without OpenStack CLI placement plugin installed:\nPLACEMENT_PUBLIC_URL=$(openstack endpoint list -c 'Service Name' -c 'Service Type' -c URL | grep placement | grep public | awk '{ print $6; }')\ncurl \"$PLACEMENT_PUBLIC_URL\"\n\n# With OpenStack CLI placement plugin installed:\nopenstack resource class list\n</code></pre>"},{"location":"openstack/planning/","title":"Planning the new deployment","text":"<p>Just like you did back when you installed your Director deployed OpenStack, the upgrade/migration to the podified OpenStack requires planning various aspects of the environment such as node roles, planning your network topology, and storage.</p> <p>In this document we cover some of this planning, but it is recommended to read the whole adoption guide before actually starting the process to be sure that there is a global understanding of the whole process.</p>"},{"location":"openstack/planning/#configurations","title":"Configurations","text":"<p>There is a fundamental difference between the Director and Operator deployments regarding the configuration of the services.</p> <p>In Director deployments many of the service configurations are abstracted by Director specific configuration options. A single Director option may trigger changes for multiple services and support for drivers (for example Cinder's) required patches to the Director code base.</p> <p>In Operator deployments this has changed to what we believe is a simpler approach: reduce the installer specific knowledge and leverage OpenShift and OpenStack service specific knowledge whenever possible.</p> <p>To this effect OpenStack services will have sensible defaults for OpenShift deployments and human operators will provide configuration snippets to provide necessary configuration, such as cinder backend configuration, or to override the defaults.</p> <p>This shortens the distance between a service specific configuration file (such as <code>cinder.conf</code>) and what the human operator provides in the manifests.</p> <p>These configuration snippets are passed to the operators in the different <code>customServiceConfig</code> sections available in the manifests, and then they are layered in the services available in the following levels. To illustrate this, if we were to set a configuration at the top Cinder level (<code>spec: cinder: template:</code>) then it would be applied to all the cinder services; for example to enable debug in all the cinder services we would do:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    template:\n      customServiceConfig: |\n        [DEFAULT]\n        debug = True\n&lt; . . . &gt;\n</code></pre> <p>If we only wanted to set it for one of the cinder services, for example the scheduler, then we would use the <code>cinderScheduler</code> section instead:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    template:\n      cinderScheduler:\n        customServiceConfig: |\n          [DEFAULT]\n          debug = True\n&lt; . . . &gt;\n</code></pre> <p>In openshift it is not recommended to store sensitive information like the credentials to the cinder storage array in the CRs, so most OpenStack operators have a mechanism to use OpenShift's <code>Secrets</code> for sensitive configuration parameters of the services and then use then by reference in the <code>customServiceConfigSecrets</code> section which is analogous to the <code>customServiceConfig</code>.</p> <p>The contents of the <code>Secret</code> references passed in the <code>customServiceConfigSecrets</code> will have the same format as <code>customServiceConfig</code>: a snippet with the section/s and configuration options.</p> <p>When there are sensitive information in the service configuration then it becomes a matter of personal preference whether to store all the configuration in the <code>Secret</code> or only the sensitive parts, but remember that if we split the configuration between <code>Secret</code> and <code>customServiceConfig</code> we still need the section header (eg: <code>[DEFAULT]</code>) to be present in both places.</p> <p>Attention should be paid to each service's adoption process as they may have some particularities regarding their configuration.</p>"},{"location":"openstack/planning/#configuration-tooling","title":"Configuration tooling","text":"<p>In order to help users to handle the configuration for the TripleO and Openstack services the tool: https://github.com/openstack-k8s-operators/os-diff has been develop to compare the configuration files between the TripleO deployment and the next gen cloud. Make sure Golang is installed and configured on your env:</p> <pre><code>git clone https://github.com/openstack-k8s-operators/os-diff\npushd os-diff\nmake build\n</code></pre> <p>Then configure ansible.cfg and ssh-config file according to your environment:</p> <pre><code>Host *\n    IdentitiesOnly yes\n\nHost virthost\n    Hostname virthost\n    IdentityFile ~/.ssh/id_rsa\n    User root\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n\n\nHost standalone\n    Hostname standalone\n    IdentityFile ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa\n    User root\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n\nHost crc\n    Hostname crc\n    IdentityFile ~/.ssh/id_rsa\n    User stack\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n</code></pre> <p>And test your connection:</p> <pre><code>ssh -F ssh.config standalone\n</code></pre>"},{"location":"openstack/planning/#node-roles","title":"Node Roles","text":"<p>In Director deployments we had 4 different standard roles for the nodes: <code>Controller</code>, <code>Compute</code>, <code>Ceph Storage</code>, <code>Swift Storage</code>, but in podified OpenStack we just make a distinction based on where things are running, in OpenShift or external to it.</p> <p>When adopting a Director OpenStack your <code>Compute</code> nodes will directly become external nodes, so there should not be much additional planning needed there.</p> <p>In many deployments being adopted the <code>Controller</code> nodes will require some thought because we'll have many OpenShift nodes where the controller services could run, and we have to decide which ones we want to use, how we are going to use them, and make sure those nodes are ready to run the services.</p> <p>In most deployments running OpenStack services on <code>master</code> nodes can have a seriously adverse impact on the OpenShift cluster, so we recommend placing OpenStack services on non <code>master</code> nodes.</p> <p>By default OpenStack Operators deploy OpenStack services on any worker node, but that is not necessarily what's best for all deployments, and there may be even services that won't even work deployed like that.</p> <p>When planing a deployment it's good to remember that not all the services on an OpenStack deployments are the same as they have very different requirements.</p> <p>Looking at the Cinder component we can clearly see different requirements for its services: the cinder-scheduler is a very light service with low memory, disk, network, and CPU usage; cinder-api service has a higher network usage due to resource listing requests; the cinder-volume service will have a high disk and network usage since many of its operations are in the data path (offline volume migration, create volume from image, etc.), and then we have the cinder-backup service which has high memory, network, and CPU (to compress data) requirements.</p> <p>We also have the Glance and Swift components that are in the data path, and let's not forget RabbitMQ and Galera services.</p> <p>Given these requirements it may be preferable not to let these services wander all over your OpenShift worker nodes with the possibility of impacting other workloads, or maybe you don't mind the light services wandering around but you want to pin down the heavy ones to a set of infrastructure nodes.</p> <p>There are also hardware restrictions to take into consideration, because if we are using a Fibre Channel (FC) Cinder backend we'll need the cinder-volume, cinder-backup, and maybe even the glance (if it's using Cinder as a backend) services to run on a OpenShift host that has an HBA.</p> <p>The OpenStack Operators allow a great deal of flexibility on where to run the OpenStack services, as we can use node labels to define which OpenShift nodes are eligible to run the different OpenStack services.  Refer to the Node Selector guide to learn more about using labels to define placement of the OpenStack services.</p> <p>TODO: Talk about Ceph Storage and Swift Storage nodes, HCI deployments, etc.</p>"},{"location":"openstack/planning/#network","title":"Network","text":"<p>TODO: Write about isolated networks, NetworkAttachmentDefinition, NetworkAttachmets, etc</p>"},{"location":"openstack/planning/#storage","title":"Storage","text":"<p>When looking into the storage in an OpenStack deployment we can differentiate 2 different kinds, the storage requirements of the services themselves and the storage used for the OpenStack users that thee services will manage.</p> <p>These requirements may drive our OpenShift node selection, as mentioned above, and may even require us to do some preparations on the OpenShift nodes before we can deploy the services.</p> <p>TODO: Galera, RabbitMQ, Swift, Glance, etc.</p>"},{"location":"openstack/planning/#cinder-requirements","title":"Cinder requirements","text":"<p>The Cinder service has both local storage used by the service and OpenStack user requirements.</p> <p>Local storage is used for example when downloading a glance image for the create volume from image operation, which can become considerable when having concurrent operations and not using cinder volume cache.</p> <p>In the Operator deployed OpenStack we now have an easy way to configure the location of the conversion directory to be an NFS share (using the extra volumes feature), something that needed to be done manually before.</p> <p>Even if it's an adoption and it may seem that there's nothing to consider regarding the Cinder backends, because we'll just be using the same ones we are using in our current deployment, we should still evaluate it, because it may not be so straightforward.</p> <p>First we need to check the transport protocol the Cinder backends are using: RBD, iSCSI, FC, NFS, NVMe-oF, etc.</p> <p>Once we know all the transport protocols we are using, we can proceed to make sure we are taking them into consideration when placing the Cinder services (as mentioned above in the Node Roles section) and the right storage transport related binaries are running on the OpenShift nodes.</p> <p>Detailed information about the specifics for each storage transport protocol can be found in the Cinder Adoption section.  Please take a good look at that document before proceeding to be able to plan the adoption better.</p>"},{"location":"openstack/pull_openstack_configuration/","title":"Pull Openstack configuration","text":"<p>Before starting to adoption workflow, we can start by pulling the configuration from the Openstack services and TripleO on our file system in order to backup the configuration files and then use it for later, during the configuration of the adopted services and for the record to compare and make sure nothing has been missed or misconfigured.</p> <p>Make sure you have pull the os-diff repository and configure according to your environment: Configure os-diff</p>"},{"location":"openstack/pull_openstack_configuration/#pull-configuration-from-a-tripleo-deployment","title":"Pull configuration from a TripleO deployment","text":"<p>Once you make sure the ssh connnection is confugred correctly and os-diff has been built, you can start to pull configuration from your Openstack services.</p> <p>All the services are describes in an Ansible role:</p> <p>collect_config vars</p> <p>Once you enabled the services you need (you can enable everything even if a services is not deployed) you can start to pull the Openstack services configuration files:</p> <pre><code>pushd os-diff\n./os-diff pull --cloud_engine=podman\n</code></pre> <p>The configuration will be pulled and stored in: <pre><code>/tmp/collect_tripleo_configs\n</code></pre></p> <p>And you provided another path with:</p> <pre><code>./os-diff pull --cloud_engine=podman -e local_working_dir=$HOME\n</code></pre> <p>Once the ansible playbook has been run, you should have into your local directory a directory per services</p> <pre><code>  \u25be tmp/\n    \u25be collect_tripleo_configs/\n      \u25be glance/\n</code></pre>"},{"location":"openstack/stop_openstack_services/","title":"Stop OpenStack services","text":"<p>Before we can start with the adoption we need to make sure that the OpenStack services have been stopped.</p> <p>This is an important step to avoid inconsistencies in the data migrated for the data-plane adoption procedure caused by resource changes after the DB has been copied to the new deployment.</p> <p>Some services are easy to stop because they only perform short asynchronous operations, but other services are a bit more complex to gracefully stop because they perform synchronous or long running operations that we may want to complete instead of aborting them.</p> <p>Since gracefully stopping all services is non-trivial and beyond the scope of this guide we'll proceed with the force method but present a couple of recommendations on how to check some things in the services.</p>"},{"location":"openstack/stop_openstack_services/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative and refer to a single node standalone director deployment, use values that are correct for your environment:</p> <pre><code>CONTROLLER1_SSH=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\"\nCONTROLLER2_SSH=\"\"\nCONTROLLER3_SSH=\"\"\n</code></pre> <p>We chose to use these ssh variables with the ssh commands instead of using ansible to try to create instructions that are independent on where they are running, but ansible commands could be used to achieve the same result if we are in the right host, for example to stop a service:</p> <pre><code>. stackrc ansible -i $(which tripleo-ansible-inventory) Controller -m shell -a \"sudo systemctl stop tripleo_horizon.service\" -b\n</code></pre>"},{"location":"openstack/stop_openstack_services/#pre-checks","title":"Pre-checks","text":"<p>We can stop OpenStack services at any moment, but we may leave things in an undesired state, so at the very least we should have a look to confirm that there are no long running operations that require other services.</p> <p>Ensure that there are no ongoing instance live migrations, volume migrations (online or offline), volume creation, backup restore, attaching, detaching, etc.</p> <pre><code>openstack server list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|'\nopenstack volume list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|'| grep -vi error\nopenstack volume backup list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|' | grep -vi error\nopenstack image list -c ID -c Status |grep -E '\\| .+ing \\|'\n</code></pre>"},{"location":"openstack/stop_openstack_services/#stopping-control-plane-services","title":"Stopping control plane services","text":"<p>We can stop OpenStack services at any moment, but we may leave things in an undesired state, so at the very least we should have a look to confirm that there are no ongoing  operations.</p> <p>1- Connect to all the controller nodes. 2- Stop the services. 3- Make sure all the services are stopped.</p> <p>The cinder-backup service on OSP 17.1 could be running as Active-Passive under pacemaker or as Active-Active, so we'll have to check how it's running and stop it.</p> <p>These steps can be automated with a simple script that relies on the previously defined environmental variables and function:</p> <pre><code># Update the services list to be stopped\nServicesToStop=(\"tripleo_horizon.service\"\n                \"tripleo_keystone.service\"\n                \"tripleo_cinder_api.service\"\n                \"tripleo_cinder_api_cron.service\"\n                \"tripleo_cinder_scheduler.service\"\n                \"tripleo_cinder_backup.service\"\n                \"tripleo_glance_api.service\"\n                \"tripleo_neutron_api.service\"\n                \"tripleo_nova_api.service\"\n                \"tripleo_placement_api.service\")\n\nPacemakerResourcesToStop=(\"openstack-cinder-volume\"\n                          \"openstack-cinder-backup\")\n\necho \"Stopping systemd OpenStack services\"\nfor service in ${ServicesToStop[*]}; do\n    for i in {1..3}; do\n        SSH_CMD=CONTROLLER${i}_SSH\n        if [ ! -z \"${!SSH_CMD}\" ]; then\n            echo \"Stopping the $service in controller $i\"\n            if ${!SSH_CMD} sudo systemctl is-active $service; then\n                ${!SSH_CMD} sudo systemctl stop $service\n            fi\n        fi\n    done\ndone\n\necho \"Checking systemd OpenStack services\"\nfor service in ${ServicesToStop[*]}; do\n    for i in {1..3}; do\n        SSH_CMD=CONTROLLER${i}_SSH\n        if [ ! -z \"${!SSH_CMD}\" ]; then\n            echo \"Checking status of $service in controller $i\"\n            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive &gt;/dev/null; then\n               echo \"ERROR: Service $service still running on controller $i\"\n            fi\n        fi\n    done\ndone\n\necho \"Stopping pacemaker OpenStack services\"\nfor i in {1..3}; do\n    SSH_CMD=CONTROLLER${i}_SSH\n    if [ ! -z \"${!SSH_CMD}\" ]; then\n        echo \"Using controller $i to run pacemaker commands\"\n        for resource in ${PacemakerResourcesToStop[*]}; do\n            if ${!SSH_CMD} sudo pcs resource config $resource; then\n                ${!SSH_CMD} sudo pcs resource disable $resource\n            fi\n        done\n        break\n    fi\ndone\n</code></pre>"},{"location":"openstack/troubleshooting/","title":"Troubleshooting","text":"<p>This document contains information about various issues you might face and how to solve them.</p>"},{"location":"openstack/troubleshooting/#errimagepull-due-to-missing-authentication","title":"ErrImagePull due to missing authentication","text":"<p>The deployed containers pull the images from private containers registries that can potentially return authentication errors like:</p> <pre><code>Failed to pull image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\":\nrpc error: code = Unknown desc = unable to retrieve auth token: invalid\nusername/password: unauthorized: Please login to the Red Hat Registry using\nyour Customer Portal credentials.\n</code></pre> <p>An example of a failed pod:</p> <pre><code>  Normal   Scheduled       3m40s                  default-scheduler  Successfully assigned openstack/rabbitmq-server-0 to worker0\n  Normal   AddedInterface  3m38s                  multus             Add eth0 [10.101.0.41/23] from ovn-kubernetes\n  Warning  Failed          2m16s (x6 over 3m38s)  kubelet            Error: ImagePullBackOff\n  Normal   Pulling         2m5s (x4 over 3m38s)   kubelet            Pulling image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\"\n  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Failed to pull image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\": rpc error: code  ... can be found here: https://access.redhat.com/RegistryAuthentication\n  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Error: ErrImagePull\n  Normal   BackOff         110s (x7 over 3m38s)   kubelet            Back-off pulling image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\"\n</code></pre> <p>To solve this issue we need to get a valid pull-secret from the official Red Hat console site, store this pull secret locally in a machine with access to the Kubernetes API (service node), and then run:</p> <pre><code>oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=&lt;pull_secret_location.json&gt;\n</code></pre> <p>The previous command will make available the authentication information in all the cluster's compute nodes, then trigger a new pod deployment to pull the container image with:</p> <pre><code>kubectl delete pod rabbitmq-server-0 -n openstack\n</code></pre> <p>And the pod should be able to pull the image successfully.  For more information about what container registries requires what type of authentication, check the official docs.</p>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Data Plane Adoption procedure","text":""},{"location":"#openstack-adoption","title":"OpenStack adoption","text":"<p>This is a procedure for adopting an OpenStack cloud.</p> <p>Perform the actions from the sub-documents in the following order:</p> <ul> <li> <p>Planning the new deployment</p> </li> <li> <p>Deploy podified backend services</p> </li> <li> <p>Pull Openstack configuration</p> </li> <li> <p>Stop OpenStack services</p> </li> <li> <p>Copy MariaDB data</p> </li> <li> <p>OVN adoption</p> </li> <li> <p>Keystone adoption</p> </li> <li> <p>Neutron adoption</p> </li> <li> <p>Ceph backend configuration (if applicable)</p> </li> <li> <p>Glance adoption</p> </li> <li> <p>Placement adoption</p> </li> <li> <p>Cinder adoption</p> </li> <li> <p>Horizon adoption</p> </li> <li> <p>Dataplane adoption</p> </li> <li> <p>Ironic adoption</p> </li> </ul> <p>If you face issues during adoption, check the Troubleshooting document for common problems and solutions.</p>"},{"location":"#post-openstack-ceph-adoption","title":"Post-OpenStack Ceph adoption","text":"<p>If the environment includes Ceph and some of its services are collocated on the Controller hosts (\"internal Ceph\"), then Ceph services need to be moved out of Controller hosts as the last step of the OpenStack adoption. Follow this documentation:</p> <ul> <li>Ceph RBD migration</li> <li>Ceph RGW migration</li> </ul>"},{"location":"#contributing","title":"Contributing","text":"<p>For information about contributing to the docs and how to run tests, see:</p> <ul> <li> <p>Contributing to documentation -   how to build docs locally, docs patterns and tips.</p> </li> <li> <p>Development environment -   how set up a local development environment where Adoption can be   executed (either manually or via the test suite).</p> </li> <li> <p>Tests -   information about the test suite and how to run it.</p> </li> </ul>"},{"location":"ceph/ceph_rbd/","title":"Data Plane adoption - Ceph RBD Migration","text":"<p>In this scenario, assuming Ceph is already &gt;= 5, either for HCI or dedicated Storage nodes, the daemons living in the OpenStack control plane should be moved/migrated into the existing external RHEL nodes (typically the compute nodes for an HCI environment or dedicated storage nodes in all the remaining use cases).</p>"},{"location":"ceph/ceph_rbd/#requirements","title":"Requirements","text":"<ul> <li>Ceph is &gt;= 5 and managed by cephadm/orchestrator.</li> <li>Ceph NFS (ganesha) migrated from a TripleO based deployment to cephadm.</li> <li>Both the Ceph public and cluster networks are propagated, via TripleO, to the target nodes.</li> <li>Ceph Mons need to keep their IPs (to avoid cold migration).</li> </ul>"},{"location":"ceph/ceph_rbd/#scenario-1-migrate-mon-and-mgr-from-controller-nodes","title":"SCENARIO 1: Migrate mon and mgr from controller nodes","text":"<p>The goal of the first POC is to prove we are able to successfully drain a controller node, in terms of ceph daemons, and move them to a different node. The initial target of the POC is RBD only, which means we\u2019re going to move only mon and mgr daemons. For the purposes of this POC, we'll deploy a ceph cluster with only mon, mgrs, and osds to simulate the environment a customer will be in before starting the migration. The goal of the first POC is to ensure that: - We can keep the mon IP addresses moving them to the CephStorage nodes. - We can drain the existing controller nodes and shut them down. - We can deploy additional monitors to the existing nodes, promoting them as   _admin nodes that can be used by administrators to manage the ceph cluster   and perform day2 operations against it. - We can keep the cluster operational during the migration.</p>"},{"location":"ceph/ceph_rbd/#prerequisites","title":"Prerequisites","text":"<p>The Storage Nodes should be configured to have both storage and storage_mgmt network to make sure we can use both Ceph public and cluster networks.</p> <p>This step is the only one where the interaction with TripleO is required. From 17+ we don\u2019t have to run any stack update, however, we have commands that should be performed to run os-net-config on the bare-metal node and configure additional networks.</p> <p>Make sure the network is defined in metalsmith.yaml for the CephStorageNodes:</p> <pre><code>- name: CephStorage\n  count: 2\n  instances:\n    - hostname: oc0-ceph-0\n      name: oc0-ceph-0\n    - hostname: oc0-ceph-1\n      name: oc0-ceph-1\n  defaults:\n    networks:\n      - network: ctlplane\n        vif: true\n      - network: storage_cloud_0\n          subnet: storage_cloud_0_subnet\n      - network: storage_mgmt_cloud_0\n          subnet: storage_mgmt_cloud_0_subnet\n    network_config:\n      template: templates/single_nic_vlans/single_nic_vlans_storage.j2\n</code></pre> <p>Then run:</p> <pre><code>openstack overcloud node provision \\\n  -o overcloud-baremetal-deployed-0.yaml --stack overcloud-0 \\\n  --network-config -y --concurrency 2 /home/stack/metalsmith-0.yam\n</code></pre> <p>Verify that the storage network is running on the node:</p> <pre><code>(undercloud) [CentOS-9 - stack@undercloud ~]$ ssh heat-admin@192.168.24.14 ip -o -4 a\nWarning: Permanently added '192.168.24.14' (ED25519) to the list of known hosts.\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n</code></pre>"},{"location":"ceph/ceph_rbd/#migrate-mons-and-mgrs-on-the-two-existing-cephstorage-nodes","title":"Migrate mon(s) and mgr(s) on the two existing CephStorage nodes","text":"<p>Create a ceph spec based on the default roles with the mon/mgr on the controller nodes.</p> <pre><code>openstack overcloud ceph spec -o ceph_spec.yaml -y  \\\n   --stack overcloud-0     overcloud-baremetal-deployed-0.yaml\n</code></pre> <p>Deploy the Ceph cluster</p> <pre><code> openstack overcloud ceph deploy overcloud-baremetal-deployed-0.yaml \\\n    --stack overcloud-0 -o deployed_ceph.yaml \\\n    --network-data ~/oc0-network-data.yaml \\\n    --ceph-spec ~/ceph_spec.yaml\n</code></pre> <p>Note:</p> <p>The ceph_spec.yaml, which is the OSP-generated description of the ceph cluster, will be used, later in the process, as the basic template required by cephadm to update the status/info of the daemons.</p> <p>Check the status of the cluster:</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_OK\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-1,oc0-controller-2 (age 19m)\n    mgr: oc0-controller-0.xzgtvo(active, since 32m), standbys: oc0-controller-1.mtxohd, oc0-controller-2.ahrgsk\n    osd: 8 osds: 8 up (since 12m), 8 in (since 18m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host ls\nHOST              ADDR           LABELS          STATUS\noc0-ceph-0        192.168.24.14  osd\noc0-ceph-1        192.168.24.7   osd\noc0-controller-0  192.168.24.15  _admin mgr mon\noc0-controller-1  192.168.24.23  _admin mgr mon\noc0-controller-2  192.168.24.13  _admin mgr mon\n</code></pre> <p>The goal of the next section is to migrate the oc0-controller-{1,2} daemons into oc0-ceph-{0,1} as the very basic scenario that demonstrates we can actually make this kind of migration using cephadm.</p>"},{"location":"ceph/ceph_rbd/#migrate-oc0-controller-1-into-oc0-ceph-0","title":"Migrate oc0-controller-1 into oc0-ceph-0","text":"<p>ssh into controller-0, then</p> <p><code>cephadm shell -v /home/ceph-admin/specs:/specs</code></p> <p>ssh into ceph-0, then</p> <p><code>sudo \u201cwatch podman ps\u201d  # watch the new mon/mgr being deployed here</code></p> <p>(optional) if mgr is active in the source node, then:</p> <pre><code>ceph mgr fail &lt;mgr instance&gt;\n</code></pre> <p>From the cephadm shell, remove the labels on oc0-controller-1</p> <pre><code>    for label in mon mgr _admin; do\n           ceph orch host rm label oc0-controller-1 $label;\n    done\n</code></pre> <p>Add the missing labels to oc0-ceph-0</p> <pre><code>[ceph: root@oc0-controller-0 /]#\n&gt; for label in mon mgr _admin; do ceph orch host label add oc0-ceph-0 $label; done\nAdded label mon to host oc0-ceph-0\nAdded label mgr to host oc0-ceph-0\nAdded label _admin to host oc0-ceph-0\n</code></pre> <p>Drain and force-remove the oc0-controller-1 node</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host drain oc0-controller-1\nScheduled to remove the following daemons from host 'oc0-controller-1'\ntype                 id\n-------------------- ---------------\nmon                  oc0-controller-1\nmgr                  oc0-controller-1.mtxohd\ncrash                oc0-controller-1\n</code></pre> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch host rm oc0-controller-1 --force\nRemoved  host 'oc0-controller-1'\n\n[ceph: root@oc0-controller-0 /]# ceph orch host ls\nHOST              ADDR           LABELS          STATUS\noc0-ceph-0        192.168.24.14  osd\noc0-ceph-1        192.168.24.7   osd\noc0-controller-0  192.168.24.15  mgr mon _admin\noc0-controller-2  192.168.24.13  _admin mgr mon\n</code></pre> <p>If you have only 3 mon nodes, and the drain of the node doesn\u2019t work as expected (the containers are still there), then SSH to controller-1 and force-purge the containers in the node:</p> <pre><code>[root@oc0-controller-1 ~]# sudo podman ps\nCONTAINER ID  IMAGE                                                                                        COMMAND               CREATED         STATUS             PORTS       NAMES\n5c1ad36472bc  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mon.oc0-contro...  35 minutes ago  Up 35 minutes ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mon-oc0-controller-1\n3b14cc7bf4dd  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mgr.oc0-contro...  35 minutes ago  Up 35 minutes ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mgr-oc0-controller-1-mtxohd\n\n[root@oc0-controller-1 ~]# cephadm rm-cluster --fsid f6ec3ebe-26f7-56c8-985d-eb974e8e08e3 --force\n\n[root@oc0-controller-1 ~]# sudo podman ps\nCONTAINER ID  IMAGE       COMMAND     CREATED     STATUS      PORTS       NAMES\n</code></pre> <p>Note: cephadm rm-cluster on a node that is not part of the cluster anymore has the effect of removing all the containers and doing some cleanup on the filesystem.</p> <p>Before shutting the oc0-controller-1 down, move the IP address (on the same network) to the oc0-ceph-0 node:</p> <pre><code>mon_host = [v2:172.16.11.54:3300/0,v1:172.16.11.54:6789/0] [v2:172.16.11.121:3300/0,v1:172.16.11.121:6789/0] [v2:172.16.11.205:3300/0,v1:172.16.11.205:6789/0]\n\n[root@oc0-controller-1 ~]# ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-ex    inet 192.168.24.23/24 brd 192.168.24.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n6: vlan100    inet 192.168.100.96/24 brd 192.168.100.255 scope global vlan100\\       valid_lft forever preferred_lft forever\n7: vlan12    inet 172.16.12.154/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n8: vlan11    inet 172.16.11.121/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n9: vlan13    inet 172.16.13.178/24 brd 172.16.13.255 scope global vlan13\\       valid_lft forever preferred_lft forever\n10: vlan70    inet 172.17.0.23/20 brd 172.17.15.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n11: vlan1    inet 192.168.24.23/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n12: vlan14    inet 172.16.14.223/24 brd 172.16.14.255 scope global vlan14\\       valid_lft forever preferred_lft forever\n</code></pre> <p>On the oc0-ceph-0:</p> <pre><code>[heat-admin@oc0-ceph-0 ~]$ ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n[heat-admin@oc0-ceph-0 ~]$ sudo ip a add 172.16.11.121 dev vlan11\n[heat-admin@oc0-ceph-0 ~]$ ip -o -4 a\n1: lo    inet 127.0.0.1/8 scope host lo\\       valid_lft forever preferred_lft forever\n5: br-storage    inet 192.168.24.14/24 brd 192.168.24.255 scope global br-storage\\       valid_lft forever preferred_lft forever\n6: vlan1    inet 192.168.24.14/24 brd 192.168.24.255 scope global vlan1\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.172/24 brd 172.16.11.255 scope global vlan11\\       valid_lft forever preferred_lft forever\n7: vlan11    inet 172.16.11.121/32 scope global vlan11\\       valid_lft forever preferred_lft forever\n8: vlan12    inet 172.16.12.46/24 brd 172.16.12.255 scope global vlan12\\       valid_lft forever preferred_lft forever\n</code></pre> <p>Poweroff oc0-controller-1.</p> <p>Add the new mon on oc0-ceph-0 using the old IP address:</p> <pre><code>[ceph: root@oc0-controller-0 /]# ceph orch daemon add mon oc0-ceph-0:172.16.11.121\nDeployed mon.oc0-ceph-0 on host 'oc0-ceph-0'\n</code></pre> <p>Check the new container in the oc0-ceph-0 node:</p> <pre><code>b581dc8bbb78  quay.io/ceph/daemon@sha256:320c364dcc8fc8120e2a42f54eb39ecdba12401a2546763b7bef15b02ce93bc4  -n mon.oc0-ceph-0...  24 seconds ago  Up 24 seconds ago              ceph-f6ec3ebe-26f7-56c8-985d-eb974e8e08e3-mon-oc0-ceph-0\n</code></pre> <p>On the cephadm shell, backup the existing ceph_spec.yaml, edit the spec removing any oc0-controller-1 entry, and replacing it with oc0-ceph-0:</p> <pre><code>cp ceph_spec.yaml ceph_spec.yaml.bkp # backup the ceph_spec.yaml file\n\n[ceph: root@oc0-controller-0 specs]# diff -u ceph_spec.yaml.bkp ceph_spec.yaml\n\n--- ceph_spec.yaml.bkp  2022-07-29 15:41:34.516329643 +0000\n+++ ceph_spec.yaml      2022-07-29 15:28:26.455329643 +0000\n@@ -7,14 +7,6 @@\n - mgr\n service_type: host\n ---\n-addr: 192.168.24.12\n-hostname: oc0-controller-1\n-labels:\n-- _admin\n-- mon\n-- mgr\n-service_type: host\n----\n addr: 192.168.24.19\n hostname: oc0-controller-2\n labels:\n@@ -38,7 +30,7 @@\n placement:\n   hosts:\n   - oc0-controller-0\n-  - oc0-controller-1\n+  - oc0-ceph-0\n   - oc0-controller-2\n service_id: mon\n service_name: mon\n@@ -47,8 +39,8 @@\n placement:\n   hosts:\n   - oc0-controller-0\n-  - oc0-controller-1\n   - oc0-controller-2\n+  - oc0-ceph-0\n service_id: mgr\n service_name: mgr\n service_type: mgr\n</code></pre> <p>Apply the resulting spec:</p> <pre><code>ceph orch apply -i ceph_spec.yaml \n\n The result of 12 is having a new mgr deployed on the oc0-ceph-0 node, and the spec reconciled within cephadm\n\n[ceph: root@oc0-controller-0 specs]# ceph orch ls\nNAME                     PORTS  RUNNING  REFRESHED  AGE  PLACEMENT\ncrash                               4/4  5m ago     61m  *\nmgr                                 3/3  5m ago     69s  oc0-controller-0;oc0-ceph-0;oc0-controller-2\nmon                                 3/3  5m ago     70s  oc0-controller-0;oc0-ceph-0;oc0-controller-2\nosd.default_drive_group               8  2m ago     69s  oc0-ceph-0;oc0-ceph-1\n\n[ceph: root@oc0-controller-0 specs]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_WARN\n            1 stray host(s) with 1 daemon(s) not managed by cephadm\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-2,oc0-ceph-0 (age 5m)\n    mgr: oc0-controller-0.xzgtvo(active, since 62m), standbys: oc0-controller-2.ahrgsk, oc0-ceph-0.hccsbb\n    osd: 8 osds: 8 up (since 42m), 8 in (since 49m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <p>Fix the warning by refreshing the mgr:</p> <pre><code>ceph mgr fail oc0-controller-0.xzgtvo\n</code></pre> <p>And at this point the cluster is clean:</p> <pre><code>[ceph: root@oc0-controller-0 specs]# ceph -s\n  cluster:\n    id:     f6ec3ebe-26f7-56c8-985d-eb974e8e08e3\n    health: HEALTH_OK\n\n  services:\n    mon: 3 daemons, quorum oc0-controller-0,oc0-controller-2,oc0-ceph-0 (age 7m)\n    mgr: oc0-controller-2.ahrgsk(active, since 25s), standbys: oc0-controller-0.xzgtvo, oc0-ceph-0.hccsbb\n    osd: 8 osds: 8 up (since 44m), 8 in (since 50m); 1 remapped pgs\n\n  data:\n    pools:   1 pools, 1 pgs\n    objects: 0 objects, 0 B\n    usage:   43 MiB used, 400 GiB / 400 GiB avail\n    pgs:     1 active+clean\n</code></pre> <p>oc0-controller-1 has been removed and powered off without leaving traces on the ceph cluster.</p> <p>The same approach and the same steps can be applied to migrate oc0-controller-2 to oc0-ceph-1.</p>"},{"location":"ceph/ceph_rbd/#screen-recording","title":"Screen Recording:","text":"<ul> <li>Externalize a TripleO deployed Ceph cluster</li> </ul>"},{"location":"ceph/ceph_rbd/#whats-next","title":"What\u2019s next","text":""},{"location":"ceph/ceph_rbd/#useful-resources","title":"Useful resources","text":"<ul> <li>cephadm - deploy additional mon(s)</li> </ul>"},{"location":"ceph/ceph_rgw/","title":"Data Plane adoption - Ceph RGW Migration","text":"<p>In this scenario, assuming Ceph is already &gt;= 5, either for HCI or dedicated Storage nodes, the RGW daemons living in the OpenStack Controller nodes will be migrated into the existing external RHEL nodes (typically the Compute nodes for an HCI environment or CephStorage nodes in the remaining use cases).</p>"},{"location":"ceph/ceph_rgw/#requirements","title":"Requirements","text":"<ul> <li>Ceph is &gt;= 5 and managed by cephadm/orchestrator</li> <li>An undercloud is still available: nodes and networks are managed by TripleO</li> </ul>"},{"location":"ceph/ceph_rgw/#ceph-daemon-cardinality","title":"Ceph Daemon Cardinality","text":"<p>Ceph 5+ applies strict constraints in the way daemons can be colocated within the same node. The resulting topology depends on the available hardware, as well as the amount of Ceph services present in the Controller nodes which are going to be retired. The following document describes the procedure required to migrate the RGW component (and keep an HA model using the Ceph Ingress daemon in a common TripleO scenario where Controller nodes represent the spec placement where the service is deployed. As a general rule, the number of services that can be migrated depends on the number of available nodes in the cluster. The following diagrams cover the distribution of the Ceph daemons on the CephStorage nodes where at least three nodes are required in a scenario that sees only RGW and RBD (no dashboard):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress <p>With dashboard, and without Manila at least four nodes are required (dashboard has no failover):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash dashboard/grafana osd rgw/ingress (free) <p>With dashboard and Manila 5 nodes minimum are required (and dashboard has no failover):</p> osd mon/mgr/crash rgw/ingress osd mon/mgr/crash rgw/ingress osd mon/mgr/crash mds/ganesha/ingress osd rgw/ingress mds/ganesha/ingress osd mds/ganesha/ingress dashboard/grafana"},{"location":"ceph/ceph_rgw/#current-status","title":"Current Status","text":"<pre><code>(undercloud) [stack@undercloud-0 ~]$ metalsmith list\n\n\n    +------------------------+    +----------------+\n    | IP Addresses           |    |  Hostname      |\n    +------------------------+    +----------------+\n    | ctlplane=192.168.24.25 |    | cephstorage-0  |\n    | ctlplane=192.168.24.10 |    | cephstorage-1  |\n    | ctlplane=192.168.24.32 |    | cephstorage-2  |\n    | ctlplane=192.168.24.28 |    | compute-0      |\n    | ctlplane=192.168.24.26 |    | compute-1      |\n    | ctlplane=192.168.24.43 |    | controller-0   |\n    | ctlplane=192.168.24.7  |    | controller-1   |\n    | ctlplane=192.168.24.41 |    | controller-2   |\n    +------------------------+    +----------------+\n</code></pre> <p>SSH into <code>controller-0</code> and check the <code>pacemaker</code> status: this will help identify the relevant information that we need to know before starting the RGW migration.</p> <pre><code>Full List of Resources:\n  * ip-192.168.24.46    (ocf:heartbeat:IPaddr2):        Started controller-0\n  * ip-10.0.0.103       (ocf:heartbeat:IPaddr2):        Started controller-1\n  * ip-172.17.1.129     (ocf:heartbeat:IPaddr2):        Started controller-2\n  * ip-172.17.3.68      (ocf:heartbeat:IPaddr2):        Started controller-0\n  * ip-172.17.4.37      (ocf:heartbeat:IPaddr2):        Started controller-1\n  * Container bundle set: haproxy-bundle\n\n[undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:\n    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-2\n    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-0\n    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-1\n</code></pre> <p>Use the <code>ip</code> command to identify the ranges of the storage networks.</p> <pre><code>[heat-admin@controller-0 ~]$ ip -o -4 a\n\n1: lo   inet 127.0.0.1/8 scope host lo\\     valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.45/24 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.46/32 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n7: br-ex    inet 10.0.0.122/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n8: vlan70   inet 172.17.5.22/24 brd 172.17.5.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n8: vlan70   inet 172.17.5.94/32 brd 172.17.5.255 scope global vlan70\\       valid_lft forever preferred_lft forever\n9: vlan50   inet 172.17.2.140/24 brd 172.17.2.255 scope global vlan50\\      valid_lft forever preferred_lft forever\n10: vlan30  inet 172.17.3.73/24 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n10: vlan30  inet 172.17.3.68/32 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n11: vlan20  inet 172.17.1.88/24 brd 172.17.1.255 scope global vlan20\\       valid_lft forever preferred_lft forever\n12: vlan40  inet 172.17.4.24/24 brd 172.17.4.255 scope global vlan40\\       valid_lft forever preferred_lft forever\n</code></pre> <p>In this example:</p> <ul> <li>vlan30 represents the Storage Network, where the new RGW instances should be   started on the CephStorage nodes</li> <li>br-ex represents the External Network, which is where in the current   environment, haproxy has the frontend VIP assigned</li> </ul>"},{"location":"ceph/ceph_rgw/#prerequisite-check-the-frontend-network-controller-nodes","title":"Prerequisite: check the frontend network (Controller nodes)","text":"<p>Identify the network that we previously had in haproxy and propagate it (via TripleO) to the CephStorage nodes. This network is used to reserve a new VIP that will be owned by Ceph and used as the entry point for the RGW service.</p> <p>ssh into <code>controller-0</code> and check the current HaProxy configuration until we find <code>ceph_rgw</code> section:</p> <pre><code>$ less /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg\n\n...\n...\nlisten ceph_rgw\n  bind 10.0.0.103:8080 transparent\n  bind 172.17.3.68:8080 transparent\n  mode http\n  balance leastconn\n  http-request set-header X-Forwarded-Proto https if { ssl_fc }\n  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }\n  http-request set-header X-Forwarded-Port %[dst_port]\n  option httpchk GET /swift/healthcheck\n  option httplog\n  option forwardfor\n  server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2\n  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2\n  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2\n</code></pre> <p>Double check the network used as HaProxy frontend:</p> <pre><code>[controller-0]$ ip -o -4 a\n\n...\n7: br-ex    inet 10.0.0.106/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n...\n</code></pre> <p>As described in the previous section, the check on controller-0 shows that we are exposing the services using the external network, which is not present in the CephStorage nodes, and we need to propagate it via TripleO.</p>"},{"location":"ceph/ceph_rgw/#propagate-the-haproxy-frontend-network-to-cephstorage-nodes","title":"Propagate the <code>HaProxy</code> frontend network to <code>CephStorage</code> nodes","text":"<p>Change the nic template used to define the ceph-storage network interfaces and add the new config section.</p> <pre><code>---\nnetwork_config:\n- type: interface\n  name: nic1\n  use_dhcp: false\n  dns_servers: {{ ctlplane_dns_nameservers }}\n  addresses:\n  - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}\n  routes: {{ ctlplane_host_routes }}\n- type: vlan\n  vlan_id: {{ storage_mgmt_vlan_id }}\n  device: nic1\n  addresses:\n  - ip_netmask: {{ storage_mgmt_ip }}/{{ storage_mgmt_cidr }}\n  routes: {{ storage_mgmt_host_routes }}\n- type: interface\n  name: nic2\n  use_dhcp: false\n  defroute: false\n- type: vlan\n  vlan_id: {{ storage_vlan_id }}\n  device: nic2\n  addresses:\n  - ip_netmask: {{ storage_ip }}/{{ storage_cidr }}\n  routes: {{ storage_host_routes }}\n- type: ovs_bridge\n  name: {{ neutron_physical_bridge_name }}\n  dns_servers: {{ ctlplane_dns_nameservers }}\n  domain: {{ dns_search_domains }}\n  use_dhcp: false\n  addresses:\n  - ip_netmask: {{ external_ip }}/{{ external_cidr }}\n  routes: {{ external_host_routes }}\n  members:\n  - type: interface\n    name: nic3\n    primary: true\n</code></pre> <p>In addition, add the External Network to the <code>baremetal.yaml</code> file used by metalsmith and run the <code>overcloud node provision</code> command passing the <code>--network-config</code> option:</p> <pre><code>- name: CephStorage\n  count: 3\n  hostname_format: cephstorage-%index%\n  instances:\n  - hostname: cephstorage-0\n  name: ceph-0\n  - hostname: cephstorage-1\n  name: ceph-1\n  - hostname: cephstorage-2\n  name: ceph-2\n  defaults:\n  profile: ceph-storage\n  network_config:\n      template: /home/stack/composable_roles/network/nic-configs/ceph-storage.j2\n  networks:\n  - network: ctlplane\n      vif: true\n  - network: storage\n  - network: storage_mgmt\n  - network: external\n</code></pre> <pre><code>(undercloud) [stack@undercloud-0]$\n\nopenstack overcloud node provision\n   -o overcloud-baremetal-deployed-0.yaml\n   --stack overcloud\n   --network-config -y\n  $PWD/network/baremetal_deployment.yaml\n</code></pre> <p>Check the new network on the <code>CephStorage</code> nodes:</p> <pre><code>[root@cephstorage-0 ~]# ip -o -4 a\n\n1: lo   inet 127.0.0.1/8 scope host lo\\     valid_lft forever preferred_lft forever\n2: enp1s0   inet 192.168.24.54/24 brd 192.168.24.255 scope global enp1s0\\       valid_lft forever preferred_lft forever\n11: vlan40  inet 172.17.4.43/24 brd 172.17.4.255 scope global vlan40\\       valid_lft forever preferred_lft forever\n12: vlan30  inet 172.17.3.23/24 brd 172.17.3.255 scope global vlan30\\       valid_lft forever preferred_lft forever\n14: br-ex   inet 10.0.0.133/24 brd 10.0.0.255 scope global br-ex\\       valid_lft forever preferred_lft forever\n</code></pre> <p>And now it\u2019s time to start migrating the RGW backends and build the ingress on top of them.</p>"},{"location":"ceph/ceph_rgw/#migrate-the-rgw-backends","title":"Migrate the RGW backends","text":"<p>To match the cardinality diagram we use cephadm labels to refer to a group of nodes where a given daemon type should be deployed.</p> <p>Add the RGW label to the cephstorage nodes:</p> <pre><code>for i in 0 1 2; {\n    ceph orch host label add cephstorage-$i rgw;\n}\n</code></pre> <pre><code>[ceph: root@controller-0 /]#\n\nfor i in 0 1 2; {\n    ceph orch host label add cephstorage-$i rgw;\n}\n\nAdded label rgw to host cephstorage-0\nAdded label rgw to host cephstorage-1\nAdded label rgw to host cephstorage-2\n\n[ceph: root@controller-0 /]# ceph orch host ls\n\nHOST        ADDR        LABELS          STATUS\ncephstorage-0  192.168.24.54  osd rgw\ncephstorage-1  192.168.24.44  osd rgw\ncephstorage-2  192.168.24.30  osd rgw\ncontroller-0   192.168.24.45  _admin mon mgr\ncontroller-1   192.168.24.11  _admin mon mgr\ncontroller-2   192.168.24.38  _admin mon mgr\n\n6 hosts in cluster\n</code></pre> <p>During the overcloud deployment, RGW is applied at step2 (external_deployment_steps), and a cephadm compatible spec is generated in <code>/home/ceph-admin/specs/rgw</code> from the ceph_mkspec ansible module. Find and patch the RGW spec, specifying the right placement using the labels approach, and change the rgw backend port to 8090 to avoid conflicts with the Ceph Ingress Daemon (*)</p> <pre><code>[root@controller-0 heat-admin]# cat rgw\n\nnetworks:\n- 172.17.3.0/24\nplacement:\n  hosts:\n  - controller-0\n  - controller-1\n  - controller-2\nservice_id: rgw\nservice_name: rgw.rgw\nservice_type: rgw\nspec:\n  rgw_frontend_port: 8080\n  rgw_realm: default\n  rgw_zone: default\n</code></pre> <p>Patch the spec replacing controller nodes with the label key</p> <pre><code>---\nnetworks:\n- 172.17.3.0/24\nplacement:\n  label: rgw\nservice_id: rgw\nservice_name: rgw.rgw\nservice_type: rgw\nspec:\n  rgw_frontend_port: 8090\n  rgw_realm: default\n  rgw_zone: default\n</code></pre> <p>(*) cephadm_check_port</p> <p>Apply the new RGW spec using the orchestrator CLI:</p> <pre><code>$ cephadm shell -m /home/ceph-admin/specs/rgw\n$ cephadm shell -- ceph orch apply -i /mnt/rgw\n</code></pre> <p>Which triggers the redeploy:</p> <pre><code>...\nosd.9                       cephstorage-2\nrgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090   starting\nrgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090   starting\nrgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090   starting\nrgw.rgw.controller-1.eyvrzw   controller-1   172.17.3.146:8080  running (5h)\nrgw.rgw.controller-2.navbxa   controller-2   172.17.3.66:8080   running (5h)\n\n...\nosd.9                       cephstorage-2\nrgw.rgw.cephstorage-0.wsjlgx  cephstorage-0  172.17.3.23:8090  running (19s)\nrgw.rgw.cephstorage-1.qynkan  cephstorage-1  172.17.3.26:8090  running (16s)\nrgw.rgw.cephstorage-2.krycit  cephstorage-2  172.17.3.81:8090  running (13s)\n</code></pre> <p>At this point, we need to make sure that the new RGW backends are reachable on the new ports, but we\u2019re going to enable an IngressDaemon on port 8080 later in the process. For this reason, ssh on each RGW node (the CephStorage nodes) and add the iptables rule to allow connections to both 8080 and 8090 ports in the CephStorage nodes.</p> <pre><code>iptables -I INPUT -p tcp -m tcp --dport 8080 -m conntrack --ctstate NEW -m comment --comment \"ceph rgw ingress\" -j ACCEPT\n\niptables -I INPUT -p tcp -m tcp --dport 8090 -m conntrack --ctstate NEW -m comment --comment \"ceph rgw backends\" -j ACCEPT\n\nfor port in 8080 8090; { \n    for i in 25 10 32; {\n       ssh heat-admin@192.168.24.$i sudo iptables -I INPUT \\\n       -p tcp -m tcp --dport $port -m conntrack --ctstate NEW \\\n       -j ACCEPT;\n   }\n}\n</code></pre> <p>From a Controller node (e.g. controller-0) try to reach (curl) the rgw backends:</p> <pre><code>for i in 26 23 81; do {\n    echo \"----\"\n    curl 172.17.3.$i:8090;\n    echo \"----\"\n    echo\ndone\n</code></pre> <p>And you should observe the following:</p> <pre><code>----\nQuery 172.17.3.23\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n\n----\nQuery 172.17.3.26\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n\n----\nQuery 172.17.3.81\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;\n---\n</code></pre>"},{"location":"ceph/ceph_rgw/#note","title":"NOTE","text":"<p>In case RGW backends are migrated in the CephStorage nodes, there\u2019s no \u201cinternalAPI\u201d network(this is not true in the case of HCI). Reconfig the RGW keystone endpoint, pointing to the external Network that has been propagated (see the previous section)</p> <pre><code>[ceph: root@controller-0 /]# ceph config dump | grep keystone\nglobal   basic rgw_keystone_url  http://172.16.1.111:5000\n\n[ceph: root@controller-0 /]# ceph config set global rgw_keystone_url http://10.0.0.103:5000\n</code></pre>"},{"location":"ceph/ceph_rgw/#deploy-a-ceph-ingressdaemon","title":"Deploy a Ceph IngressDaemon","text":"<p><code>HaProxy</code> is managed by TripleO via <code>Pacemaker</code>: the three running instances at this point will point to the old RGW backends, resulting in a wrong, not working configuration. Since we\u2019re going to deploy the Ceph Ingress Daemon, the first thing to do is remove the existing <code>ceph_rgw</code> config, clean up the config created by TripleO and restart the service to make sure other services are not affected by this change.</p> <p>ssh  on each Controller node and remove the following is the section from <code>/var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg</code>:</p> <pre><code>listen ceph_rgw\n  bind 10.0.0.103:8080 transparent\n  mode http\n  balance leastconn\n  http-request set-header X-Forwarded-Proto https if { ssl_fc }\n  http-request set-header X-Forwarded-Proto http if !{ ssl_fc }\n  http-request set-header X-Forwarded-Port %[dst_port]\n  option httpchk GET /swift/healthcheck\n  option httplog\n  option forwardfor\n   server controller-0.storage.redhat.local 172.17.3.73:8080 check fall 5 inter 2000 rise 2\n  server controller-1.storage.redhat.local 172.17.3.146:8080 check fall 5 inter 2000 rise 2\n  server controller-2.storage.redhat.local 172.17.3.156:8080 check fall 5 inter 2000 rise 2\n</code></pre> <p>Restart <code>haproxy-bundle</code> and make sure it\u2019s started:</p> <pre><code>[root@controller-0 ~]# sudo pcs resource restart haproxy-bundle\nhaproxy-bundle successfully restarted\n\n\n[root@controller-0 ~]# sudo pcs status | grep haproxy\n\n  * Container bundle set: haproxy-bundle [undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp17-openstack-haproxy:pcmklatest]:\n    * haproxy-bundle-podman-0   (ocf:heartbeat:podman):  Started controller-0\n    * haproxy-bundle-podman-1   (ocf:heartbeat:podman):  Started controller-1\n    * haproxy-bundle-podman-2   (ocf:heartbeat:podman):  Started controller-2\n</code></pre> <p>Double check no process is bound to 8080 anymore\u201d</p> <pre><code>[root@controller-0 ~]# ss -antop | grep 8080\n[root@controller-0 ~]#\n</code></pre> <p>And the swift CLI should fail at this point:</p> <pre><code>(overcloud) [root@cephstorage-0 ~]# swift list\n\nHTTPConnectionPool(host='10.0.0.103', port=8080): Max retries exceeded with url: /swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json (Caused by NewConnectionError('&lt;urllib3.connection.HTTPConnection object at 0x7fc41beb0430&gt;: Failed to establish a new connection: [Errno 111] Connection refused'))\n</code></pre> <p>Now we can start deploying the Ceph IngressDaemon on the CephStorage nodes.</p> <p>Set the required images for both HaProxy and Keepalived</p> <pre><code>[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_haproxy quay.io/ceph/haproxy:2.3\n\n[ceph: root@controller-0 /]# ceph config set mgr mgr/cephadm/container_image_keepalived quay.io/ceph/keepalived:2.1.5\n</code></pre> <p>Prepare the ingress spec and mount it to cephadm:</p> <pre><code>$ sudo vim /home/ceph-admin/specs/rgw_ingress\n</code></pre> <p>and paste the following content:</p> <pre><code>---\nservice_type: ingress\nservice_id: rgw.rgw\nplacement:\n  label: rgw\nspec:\n  backend_service: rgw.rgw\n  virtual_ip: 10.0.0.89/24\n  frontend_port: 8080\n  monitor_port: 8898\n  virtual_interface_networks:\n    - 10.0.0.0/24\n</code></pre> <p>Mount the generated spec and apply it using the orchestrator CLI:</p> <pre><code>$ cephadm shell -m /home/ceph-admin/specs/rgw_ingress\n$ cephadm shell -- ceph orch apply -i /mnt/rgw_ingress\n</code></pre> <p>Wait until the ingress is deployed and query the resulting endpoint:</p> <pre><code>[ceph: root@controller-0 /]# ceph orch ls\n\nNAME                    PORTS               RUNNING  REFRESHED  AGE  PLACEMENT\ncrash                                           6/6  6m ago     3d   *\ningress.rgw.rgw         10.0.0.89:8080,8898     6/6  37s ago    60s  label:rgw\nmds.mds                   3/3  6m ago   3d   controller-0;controller-1;controller-2\nmgr                       3/3  6m ago   3d   controller-0;controller-1;controller-2\nmon                       3/3  6m ago   3d   controller-0;controller-1;controller-2\nosd.default_drive_group   15  37s ago   3d   cephstorage-0;cephstorage-1;cephstorage-2\nrgw.rgw   ?:8090          3/3  37s ago  4m   label:rgw\n</code></pre> <pre><code>[ceph: root@controller-0 /]# curl  10.0.0.89:8080\n\n---\n&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;&lt;ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult&gt;[ceph: root@controller-0 /]#\n\u2014\n</code></pre> <p>The result above shows that we\u2019re able to reach the backend from the IngressDaemon, which means we\u2019re almost ready to interact with it using the swift CLI.</p>"},{"location":"ceph/ceph_rgw/#update-the-object-store-endpoints","title":"Update the object-store endpoints","text":"<p>The endpoints still point to the old VIP owned by pacemaker, but given it\u2019s still used by other services and we reserved a new VIP on the same network, before any other action we should update the object-store endpoint.</p> <p>List the current endpoints:</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object\n\n| 1326241fb6b6494282a86768311f48d1 | regionOne | swift      | object-store   | True | internal  | http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |\n| 8a34817a9d3443e2af55e108d63bb02b | regionOne | swift      | object-store   | True | public    | http://10.0.0.103:8080/swift/v1/AUTH_%(project_id)s  |\n| fa72f8b8b24e448a8d4d1caaeaa7ac58 | regionOne | swift      | object-store   | True | admin     | http://172.17.3.68:8080/swift/v1/AUTH_%(project_id)s |\n</code></pre> <p>Update the endpoints pointing to the Ingress VIP:</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ openstack endpoint set --url \"http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s\" 95596a2d92c74c15b83325a11a4f07a3\n\n(overcloud) [stack@undercloud-0 ~]$ openstack endpoint list | grep object-store\n| 6c7244cc8928448d88ebfad864fdd5ca | regionOne | swift      | object-store   | True | internal  | http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |\n| 95596a2d92c74c15b83325a11a4f07a3 | regionOne | swift      | object-store   | True | public    | http://10.0.0.89:8080/swift/v1/AUTH_%(project_id)s   |\n| e6d0599c5bf24a0fb1ddf6ecac00de2d | regionOne | swift      | object-store   | True | admin     | http://172.17.3.79:8080/swift/v1/AUTH_%(project_id)s |\n</code></pre> <p>And repeat the same action for both internal and admin. Test the migrated service.</p> <pre><code>(overcloud) [stack@undercloud-0 ~]$ swift list --debug\n\nDEBUG:swiftclient:Versionless auth_url - using http://10.0.0.115:5000/v3 as endpoint\nDEBUG:keystoneclient.auth.identity.v3.base:Making authentication request to http://10.0.0.115:5000/v3/auth/tokens\nDEBUG:urllib3.connectionpool:Starting new HTTP connection (1): 10.0.0.115:5000\nDEBUG:urllib3.connectionpool:http://10.0.0.115:5000 \"POST /v3/auth/tokens HTTP/1.1\" 201 7795\nDEBUG:keystoneclient.auth.identity.v3.base:{\"token\": {\"methods\": [\"password\"], \"user\": {\"domain\": {\"id\": \"default\", \"name\": \"Default\"}, \"id\": \"6f87c7ffdddf463bbc633980cfd02bb3\", \"name\": \"admin\", \"password_expires_at\": null}, \n\n\n...\n...\n...\n\nDEBUG:swiftclient:REQ: curl -i http://10.0.0.89:8080/swift/v1/AUTH_852f24425bb54fa896476af48cbe35d3?format=json -X GET -H \"X-Auth-Token: gAAAAABj7KHdjZ95syP4c8v5a2zfXckPwxFQZYg0pgWR42JnUs83CcKhYGY6PFNF5Cg5g2WuiYwMIXHm8xftyWf08zwTycJLLMeEwoxLkcByXPZr7kT92ApT-36wTfpi-zbYXd1tI5R00xtAzDjO3RH1kmeLXDgIQEVp0jMRAxoVH4zb-DVHUos\" -H \"Accept-Encoding: gzip\"\nDEBUG:swiftclient:RESP STATUS: 200 OK\nDEBUG:swiftclient:RESP HEADERS: {'content-length': '2', 'x-timestamp': '1676452317.72866', 'x-account-container-count': '0', 'x-account-object-count': '0', 'x-account-bytes-used': '0', 'x-account-bytes-used-actual': '0', 'x-account-storage-policy-default-placement-container-count': '0', 'x-account-storage-policy-default-placement-object-count': '0', 'x-account-storage-policy-default-placement-bytes-used': '0', 'x-account-storage-policy-default-placement-bytes-used-actual': '0', 'x-trans-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'x-openstack-request-id': 'tx00000765c4b04f1130018-0063eca1dd-1dcba-default', 'accept-ranges': 'bytes', 'content-type': 'application/json; charset=utf-8', 'date': 'Wed, 15 Feb 2023 09:11:57 GMT'}\nDEBUG:swiftclient:RESP BODY: b'[]'\n</code></pre> <p>Run tempest tests against object-storage:</p> <pre><code>(overcloud) [stack@undercloud-0 tempest-dir]$  tempest run --regex tempest.api.object_storage\n...\n...\n...\n======\nTotals\n======\nRan: 141 tests in 606.5579 sec.\n - Passed: 128\n - Skipped: 13\n - Expected Fail: 0\n - Unexpected Success: 0\n - Failed: 0\nSum of execute time for each test: 657.5183 sec.\n\n==============\nWorker Balance\n==============\n - Worker 0 (1 tests) =&gt; 0:10:03.400561\n - Worker 1 (2 tests) =&gt; 0:00:24.531916\n - Worker 2 (4 tests) =&gt; 0:00:10.249889\n - Worker 3 (30 tests) =&gt; 0:00:32.730095\n - Worker 4 (51 tests) =&gt; 0:00:26.246044\n - Worker 5 (6 tests) =&gt; 0:00:20.114803\n - Worker 6 (20 tests) =&gt; 0:00:16.290323\n - Worker 7 (27 tests) =&gt; 0:00:17.103827\n</code></pre>"},{"location":"ceph/ceph_rgw/#additional-resources","title":"Additional Resources","text":"<p>A screen recording is available here.</p>"},{"location":"contributing/development_environment/","title":"Development environment","text":"<p>This is a guide for an install_yamls based Adoption environment with network isolation as an alternative to the CRC and Vagrant TripleO Standalone development environment guide.</p> <p>The Adoption development environment utilizes install_yamls for CRC VM creation and for creation of the VM that hosts the original Wallaby OpenStack in Standalone configuration.</p>"},{"location":"contributing/development_environment/#environment-prep","title":"Environment prep","text":"<p>Get install_yamls:</p> <pre><code>git clone https://github.com/openstack-k8s-operators/install_yamls.git\n</code></pre> <p>Install tools for operator development:</p> <pre><code>cd ~/install_yamls/devsetup\nmake download_tools\n</code></pre>"},{"location":"contributing/development_environment/#deployment-of-crc-with-network-isolation","title":"Deployment of CRC with network isolation","text":"<pre><code>cd ~/install_yamls/devsetup\nPULL_SECRET=$HOME/pull-secret.txt CPUS=12 MEMORY=40000 DISK=100 make crc\n\neval $(crc oc-env)\noc login -u kubeadmin -p 12345678 https://api.crc.testing:6443\n\nmake crc_attach_default_interface\n</code></pre>"},{"location":"contributing/development_environment/#development-environment-with-openstack-ironic","title":"Development environment with Openstack ironic","text":"<p>Create the BMaaS network (<code>crc-bmaas</code>) and virtual baremetal nodes controlled by a RedFish BMC emulator.</p> <pre><code>cd ..  # back to install_yamls\nmake nmstate\nmake namespace\ncd devsetup  # back to install_yamls/devsetup\nmake bmaas\n</code></pre> <p>A node definition YAML file to use with the <code>openstack baremetal create &lt;file&gt;.yaml</code> command can be generated for the virtual baremetal nodes by running the <code>bmaas_generate_nodes_yaml</code> make target. Store it in a temp file for later.</p> <pre><code>make bmaas_generate_nodes_yaml | tail -n +2 | tee /tmp/ironic_nodes.yaml\n</code></pre> <p>Set variables to deploy edpm Standalone with additional network (<code>baremetal</code>) and compute driver <code>ironic</code>.</p> <pre><code>cat &lt;&lt; EOF &gt; /tmp/addtional_nets.json\n[\n  {\n    \"type\": \"network\",\n    \"name\": \"crc-bmaas\",\n    \"standalone_config\": {\n      \"type\": \"ovs_bridge\",\n      \"name\": \"baremetal\",\n      \"mtu\": 1500,\n      \"vip\": true,\n      \"ip_subnet\": \"172.20.1.0/24\",\n      \"allocation_pools\": [\n        {\n          \"start\": \"172.20.1.100\",\n          \"end\": \"172.20.1.150\"\n        }\n      ],\n      \"host_routes\": [\n        {\n          \"destination\": \"192.168.130.0/24\",\n          \"nexthop\": \"172.20.1.1\"\n        }\n      ]\n    }\n  }\n]\nEOF\nexport EDPM_COMPUTE_ADDITIONAL_NETWORKS=$(jq -c . /tmp/addtional_nets.json)\nexport STANDALONE_COMPUTE_DRIVER=ironic\nexport NTP_SERVER=pool.ntp.org  # Only neccecary if not on the RedHat network ...\nexport EDPM_COMPUTE_CEPH_ENABLED=false  # Optional\n</code></pre> <p>Use the install_yamls devsetup to create a virtual machine connected to the isolated networks.</p> <p>Create the edpm-compute-0 virtual machine.</p> <pre><code>cd install_yamls/devsetup\nmake standalone\n</code></pre>"},{"location":"contributing/development_environment/#install-the-openstack-k8s-operators-openstack-operator","title":"Install the openstack-k8s-operators (openstack-operator)","text":"<pre><code>cd ..  # back to install_yamls\nmake crc_storage\nmake input\nmake openstack\n</code></pre>"},{"location":"contributing/development_environment/#convenience-steps","title":"Convenience steps","text":"<p>To make our life easier we can copy the deployment passwords we'll be using in the backend services deployment phase of the data plane adoption.</p> <pre><code>scp -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100:/root/tripleo-standalone-passwords.yaml ~/\n</code></pre> <p>If we want to be able to easily run <code>openstack</code> commands from the host without actually installing the package and copying the configuration file from the VM we can create a simple alias:</p> <pre><code>alias openstack=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 OS_CLOUD=standalone openstack\"\n</code></pre>"},{"location":"contributing/development_environment/#route-networks","title":"Route networks","text":"<p>Route VLAN20 to have access to the MariaDB cluster:</p> <pre><code>EDPM_BRIDGE=$(sudo virsh dumpxml edpm-compute-0 | grep -oP \"(?&lt;=bridge=').*(?=')\")\nsudo ip link add link $EDPM_BRIDGE name vlan20 type vlan id 20\nsudo ip addr add dev vlan20 172.17.0.222/24\nsudo ip link set up dev vlan20\n</code></pre>"},{"location":"contributing/development_environment/#snapshotrevert","title":"Snapshot/revert","text":"<p>When the deployment of the Standalone OpenStack is finished, it's a good time to snapshot the machine, so that multiple Adoption attempts can be done without having to deploy from scratch.</p> <pre><code>cd ~/install_yamls/devsetup\nmake standalone_snapshot\n</code></pre> <p>And when you wish to revert the Standalone deployment to the snapshotted state:</p> <pre><code>cd ~/install_yamls/devsetup\nmake standalone_revert\n</code></pre> <p>Similar snapshot could be done for the CRC virtual machine, but the developer environment reset on CRC side can be done sufficiently via the install_yamls <code>*_cleanup</code> targets. This is further detailed in the section: Reset the environment to pre-adoption state</p>"},{"location":"contributing/development_environment/#create-a-workload-to-adopt","title":"Create a workload to adopt","text":""},{"location":"contributing/development_environment/#ironic-steps","title":"Ironic Steps","text":""},{"location":"contributing/development_environment/#enroll-baremetal-nodes-make-bmaas_generate_nodes_yaml-tail-n-2-tee-tmpironic_nodesyaml-scp-i-homeinstall_yamlsoutedpmansibleee-ssh-key-id_rsa-tmpironic_nodesyaml-root192168122100-ssh-i-homeinstall_yamlsoutedpmansibleee-ssh-key-id_rsa-root192168122100-export-os_cloudstandalone-openstack-baremetal-create-rootironic_nodesyaml-export-ironic_python_agent_ramdisk_idopenstack-image-show-deploy-ramdisk-c-id-f-value-export-ironic_python_agent_kernel_idopenstack-image-show-deploy-kernel-c-id-f-value-for-node-in-openstack-baremetal-node-list-c-uuid-f-value-do-openstack-baremetal-node-set-node-driver-info-deploy_ramdiskironic_python_agent_ramdisk_id-driver-info-deploy_kernelironic_python_agent_kernel_id-resource-class-baremetal-property-capabilitiesboot_modeuefi-done-create-a-baremetal-flavor-openstack-flavor-create-baremetal-ram-1024-vcpus-1-disk-15-property-resourcesvcpu0-property-resourcesmemory_mb0-property-resourcesdisk_gb0-property-resourcescustom_baremetal1-property-capabilitiesboot_modeuefi-create-image-imgfedora-cloud-base-38-16x86_64qcow2-urlhttpsdownloadfedoraprojectorgpubfedoralinuxreleases38cloudx86_64imagesimg-curl-o-tmpimg-l-url-disk_formatqemu-img-info-tmpimg-grep-file-format-awk-print-nf-openstack-image-create-container-format-bare-disk-format-disk_format-fedora-cloud-base-38-tmpimg-export-baremetal_nodesopenstack-baremetal-node-list-c-uuid-f-value-manage-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-node-manage-node-done-wait-for-nodes-to-reach-manageable-state-watch-openstack-baremetal-node-list-inspect-baremetal-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-introspection-start-node-done-wait-for-inspection-to-complete-watch-openstack-baremetal-introspection-list-provide-nodes-for-node-in-baremetal_nodes-do-openstack-baremetal-node-provide-node-done-wait-for-nodes-to-reach-available-state-watch-openstack-baremetal-node-list-create-an-instance-on-baremetal-openstack-server-show-baremetal-test-openstack-server-create-baremetal-test-flavor-baremetal-image-fedora-cloud-base-38-nic-net-idprovisioning-wait-check-instance-status-and-network-connectivity-openstack-server-show-baremetal-test-ping-c-4-openstack-server-show-baremetal-test-f-json-c-addresses-jq-r-addressesprovisioning0","title":"<pre><code># Enroll baremetal nodes\nmake bmaas_generate_nodes_yaml | tail -n +2 | tee /tmp/ironic_nodes.yaml\nscp -i $HOME/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa /tmp/ironic_nodes.yaml root@192.168.122.100:\nssh -i $HOME/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\n\nexport OS_CLOUD=standalone\nopenstack baremetal create /root/ironic_nodes.yaml\nexport IRONIC_PYTHON_AGENT_RAMDISK_ID=$(openstack image show deploy-ramdisk -c id -f value)\nexport IRONIC_PYTHON_AGENT_KERNEL_ID=$(openstack image show deploy-kernel -c id -f value)\nfor node in $(openstack baremetal node list -c UUID -f value); do\n  openstack baremetal node set $node \\\n    --driver-info deploy_ramdisk=${IRONIC_PYTHON_AGENT_RAMDISK_ID} \\\n    --driver-info deploy_kernel=${IRONIC_PYTHON_AGENT_KERNEL_ID} \\\n    --resource-class baremetal \\\n    --property capabilities='boot_mode:uefi'\ndone\n\n# Create a baremetal flavor\nopenstack flavor create baremetal --ram 1024 --vcpus 1 --disk 15 \\\n  --property resources:VCPU=0 \\\n  --property resources:MEMORY_MB=0 \\\n  --property resources:DISK_GB=0 \\\n  --property resources:CUSTOM_BAREMETAL=1 \\\n  --property capabilities:boot_mode=\"uefi\"\n\n# Create image\nIMG=Fedora-Cloud-Base-38-1.6.x86_64.qcow2\nURL=https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/$IMG\ncurl -o /tmp/${IMG} -L $URL\nDISK_FORMAT=$(qemu-img info /tmp/${IMG} | grep \"file format:\" | awk '{print $NF}')\nopenstack image create --container-format bare --disk-format ${DISK_FORMAT} Fedora-Cloud-Base-38 &lt; /tmp/${IMG}\n\nexport BAREMETAL_NODES=$(openstack baremetal node list -c UUID -f value)\n# Manage nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal node manage $node\ndone\n\n# Wait for nodes to reach \"manageable\" state\nwatch openstack baremetal node list\n\n# Inspect baremetal nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal introspection start $node\ndone\n\n# Wait for inspection to complete\nwatch openstack baremetal introspection list\n\n# Provide nodes\nfor node in $BAREMETAL_NODES; do\n  openstack baremetal node provide $node\ndone\n\n# Wait for nodes to reach \"available\" state\nwatch openstack baremetal node list\n\n# Create an instance on baremetal\nopenstack server show baremetal-test || {\n    openstack server create baremetal-test --flavor baremetal --image Fedora-Cloud-Base-38 --nic net-id=provisioning --wait\n}\n\n# Check instance status and network connectivity\nopenstack server show baremetal-test\nping -c 4 $(openstack server show baremetal-test -f json -c addresses | jq -r .addresses.provisioning[0])\n</code></pre>","text":"<pre><code>export OS_CLOUD=standalone\nsource ~/install_yamls/devsetup/scripts/edpm-deploy-instance.sh\n</code></pre> <p>Confirm the image UUID can be seen in Ceph's images pool. <pre><code>ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 sudo cephadm shell -- rbd -p images ls -l\n</code></pre></p> <p>Create a Cinder volume, a backup from it, and snapshot it. <pre><code>openstack volume create --image cirros --bootable --size 1 disk\nopenstack volume backup create --name backup disk\nopenstack volume snapshot create --volume disk snapshot\n</code></pre></p> <p>Add volume to the test VM <pre><code>openstack server add volume test disk\n</code></pre></p>"},{"location":"contributing/development_environment/#performing-the-data-plane-adoption","title":"Performing the Data Plane Adoption","text":"<p>The development environment is now set up, you can go to the Adoption documentation and perform adoption manually, or run the test suite against your environment.</p>"},{"location":"contributing/development_environment/#reset-the-environment-to-pre-adoption-state","title":"Reset the environment to pre-adoption state","text":"<p>The development environment must be rolled back in case we want to execute another Adoption run.</p> <p>Delete the data-plane and control-plane resources from the CRC vm <pre><code>oc delete osdp openstack\noc delete oscp openstack\n</code></pre></p> <p>Revert the standalone vm to the snapshotted state</p> <p><pre><code>cd ~/install_yamls/devsetup\nmake standalone_revert\n</code></pre> Clean up and initialize the storage PVs in CRC vm <pre><code>cd ..\nmake crc_storage_cleanup\nmake crc_storage\n</code></pre></p>"},{"location":"contributing/development_environment/#experimenting-with-an-additional-compute-node","title":"Experimenting with an additional compute node","text":"<p>The following is not on the critical path of preparing the development environment for Adoption, but it shows how to make the environment work with an additional compute node VM.</p> <p>The remaining steps should be completed on the hypervisor hosting crc and edpm-compute-0.</p>"},{"location":"contributing/development_environment/#deploy-ng-control-plane-with-ceph","title":"Deploy NG Control Plane with Ceph","text":"<p>Export the Ceph configuration from edpm-compute-0 into a secret. <pre><code>SSH=$(ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100)\nKEY=$($SSH \"cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0\")\nCONF=$($SSH \"cat /etc/ceph/ceph.conf | base64 -w 0\")\n\ncat &lt;&lt;EOF &gt; ceph_secret.yaml\napiVersion: v1\ndata:\n  ceph.client.openstack.keyring: $KEY\n  ceph.conf: $CONF\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\ntype: Opaque\nEOF\n\noc create -f ceph_secret.yaml\n</code></pre> Deploy the NG control plane with Ceph as backend for Glance and Cinder. As described in the install_yamls README, use the sample config located at https://github.com/openstack-k8s-operators/openstack-operator/blob/main/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml but make sure to replace the <code>_FSID_</code> in the sample with the one from the secret created in the previous step. <pre><code>curl -o /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml https://raw.githubusercontent.com/openstack-k8s-operators/openstack-operator/main/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\nFSID=$(oc get secret ceph-conf-files -o json | jq -r '.data.\"ceph.conf\"' | base64 -d | grep fsid | sed -e 's/fsid = //') &amp;&amp; echo $FSID\nsed -i \"s/_FSID_/${FSID}/\" /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\noc apply -f /tmp/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml\n</code></pre></p> <p>A NG control plane which uses the same Ceph backend should now be functional. If you create a test image on the NG system to confirm it works from the configuration above, be sure to read the warning in the next section.</p> <p>Before beginning adoption testing or development you may wish to deploy an EDPM node as described in the following section.</p>"},{"location":"contributing/development_environment/#warning-about-two-openstacks-and-one-ceph","title":"Warning about two OpenStacks and one Ceph","text":"<p>Though workloads can be created in the NG deployment to test, be careful not to confuse them with workloads from the Wallaby cluster to be migrated. The following scenario is now possible.</p> <p>A Glance image exists on the Wallaby OpenStack to be adopted. <pre><code>[stack@standalone standalone]$ export OS_CLOUD=standalone\n[stack@standalone standalone]$ openstack image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 33a43519-a960-4cd0-a593-eca56ee553aa | cirros | active |\n+--------------------------------------+--------+--------+\n[stack@standalone standalone]$\n</code></pre> If you now create an image with the NG cluster, then a Glance image will exsit on the NG OpenStack which will adopt the workloads of the wallaby. <pre><code>[fultonj@hamfast ng]$ export OS_CLOUD=default\n[fultonj@hamfast ng]$ export OS_PASSWORD=12345678\n[fultonj@hamfast ng]$ openstack image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 4ebccb29-193b-4d52-9ffd-034d440e073c | cirros | active |\n+--------------------------------------+--------+--------+\n[fultonj@hamfast ng]$\n</code></pre> Both Glance images are stored in the same Ceph pool. <pre><code>ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 sudo cephadm shell -- rbd -p images ls -l\nInferring fsid 7133115f-7751-5c2f-88bd-fbff2f140791\nUsing recent ceph image quay.rdoproject.org/tripleowallabycentos9/daemon@sha256:aa259dd2439dfaa60b27c9ebb4fb310cdf1e8e62aa7467df350baf22c5d992d8\nNAME                                       SIZE     PARENT  FMT  PROT  LOCK\n33a43519-a960-4cd0-a593-eca56ee553aa         273 B            2\n33a43519-a960-4cd0-a593-eca56ee553aa@snap    273 B            2  yes\n4ebccb29-193b-4d52-9ffd-034d440e073c       112 MiB            2\n4ebccb29-193b-4d52-9ffd-034d440e073c@snap  112 MiB            2  yes\n</code></pre> However, as far as each Glance service is concerned each has one image. Thus, in order to avoid confusion during adoption the test Glance image on the NG OpenStack should be deleted. <pre><code>openstack image delete 4ebccb29-193b-4d52-9ffd-034d440e073c\n</code></pre> Connecting the NG OpenStack to the existing Ceph cluster is part of the adoption procedure so that the data migration can be minimized but understand the implications of the above example.</p>"},{"location":"contributing/development_environment/#deploy-edpm-compute-1","title":"Deploy edpm-compute-1","text":"<p>edpm-compute-0 is not available as a standard EDPM system to be managed by edpm-ansible or dataplane-operator because it hosts the wallaby deployment which will be adopted and after adoption it will only host the Ceph server.</p> <p>Use the install_yamls devsetup to create additional virtual machines and be sure that the <code>EDPM_COMPUTE_SUFFIX</code> is set to <code>1</code> or greater. Do not set <code>EDPM_COMPUTE_SUFFIX</code> to <code>0</code> or you could delete the Wallaby system created in the previous section.</p> <p>When deploying EDPM nodes add an <code>extraMounts</code> like the following in the <code>OpenStackDataPlaneNodeSet</code> CR <code>nodeTemplate</code> so that they will be configured to use the same Ceph cluster.</p> <pre><code>    edpm-compute:\n      nodeTemplate:\n        extraMounts:\n        - extraVolType: Ceph\n          volumes:\n          - name: ceph\n            secret:\n              secretName: ceph-conf-files\n          mounts:\n          - name: ceph\n            mountPath: \"/etc/ceph\"\n            readOnly: true\n</code></pre> <p>A NG data plane which uses the same Ceph backend should now be functional. Be careful about not confusing new workloads to test the NG OpenStack with the Wallaby OpenStack as described in the previous section.</p>"},{"location":"contributing/development_environment/#begin-adoption-testing-or-development","title":"Begin Adoption Testing or Development","text":"<p>We should now have:</p> <ul> <li>An NG glance service based on Antelope running on CRC</li> <li>An TripleO-deployed glance serviced running on edpm-compute-0</li> <li>Both services have the same Ceph backend</li> <li>Each service has their own independent database</li> </ul> <p>An environment above is assumed to be available in the Glance Adoption documentation. You may now follow other Data Plane Adoption procedures described in the documentation. The same pattern can be applied to other services.</p>"},{"location":"contributing/documentation/","title":"Contributing to documentation","text":""},{"location":"contributing/documentation/#rendering-documentation-locally","title":"Rendering documentation locally","text":"<p>Install docs build requirements into virtualenv:</p> <pre><code>python3 -m venv local/docs-venv\nsource local/docs-venv/bin/activate\npip install -r docs/doc_requirements.txt\n</code></pre> <p>Serve docs site on localhost:</p> <pre><code>mkdocs serve\n</code></pre> <p>Click the link it outputs. As you save changes to files modified in your editor, the browser will automatically show the new content.</p>"},{"location":"contributing/documentation/#patterns-and-tips-for-contributing-to-documentation","title":"Patterns and tips for contributing to documentation","text":"<ul> <li> <p>Pages concerning individual components/services should make sense in   the context of the broader adoption procedure. While adopting a   service in isolation is an option for developers, let's write the   documentation with the assumption the adoption procedure is being   done in full, going step by step (one doc after another).</p> </li> <li> <p>The procedure should be written with production use in mind. This   repository could be used as a starting point for product   technical documentation. We should not tie the documentation to   something that wouldn't translate well from dev envs to production.</p> <ul> <li>This includes not assuming that the source environment is   Standalone, and the destination is CRC. We can provide examples for   Standalone/CRC, but it should be possible to use the procedure   with fuller environments in a way that is obvious from the docs.</li> </ul> </li> <li> <p>If possible, try to make code snippets copy-pastable. Use shell   variables if the snippets should be parametrized. Use <code>oc</code> rather   than <code>kubectl</code> in snippets.</p> </li> <li> <p>Focus on the \"happy path\" in the docs as much as possible,   troubleshooting info can go into the Troubleshooting page, or   alternatively a troubleshooting section at the end of the document,   visibly separated from the main procedure.</p> </li> <li> <p>The full procedure will inevitably happen to be quite long, so let's   try to be concise in writing to keep the docs consumable (but not to   a point of making things difficult to understand or omitting   important things).</p> </li> <li> <p>A bash alias can be created for long command however when implementing   them in the test roles you should transform them to avoid command not   found errors.   From:   <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\nopenstack endpoint list | grep network\n</code></pre>   TO:   <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\n${BASH_ALIASES[openstack]} endpoint list | grep network\n</code></pre></p> </li> </ul>"},{"location":"contributing/tests/","title":"Tests","text":""},{"location":"contributing/tests/#test-suite-information","title":"Test suite information","text":"<p>The adoption docs repository also includes a test suite for Adoption. There are targets in the Makefile which can be used to execute the test suite:</p> <ul> <li> <p><code>test-minimal</code> - a minimal test scenario, the eventual set of   services in this scenario should be the \"core\" services needed to   launch a VM. This scenario assumes local storage backend for   services like Glance and Cinder.</p> </li> <li> <p><code>test-with-ceph</code> - like 'minimal' but with Ceph storage backend for   Glance and Cinder.</p> </li> </ul>"},{"location":"contributing/tests/#configuring-the-test-suite","title":"Configuring the test suite","text":"<ul> <li> <p>Create <code>tests/vars.yaml</code> and <code>tests/secrets.yaml</code> by copying the   included samples (<code>tests/vars.sample.yaml</code>,   <code>tests/secrets.sample.yaml</code>).</p> </li> <li> <p>Walk through the <code>tests/vars.yaml</code> and <code>tests/secrets.yaml</code> files   and see if you need to edit any values. If you are using the   documented development environment, majority of the defaults should   work out of the box. The comments in the YAML files will guide you   regarding the expected values. You may want to double check that   these variables suit your environment:</p> </li> <li> <p><code>install_yamls_path</code></p> </li> <li> <p><code>tripleo_passwords</code></p> </li> <li> <p><code>controller*_ssh</code></p> </li> <li> <p><code>edpm_privatekey_path</code></p> </li> <li> <p><code>edpm_chrony_ntp_servers</code></p> </li> </ul>"},{"location":"contributing/tests/#running-the-tests","title":"Running the tests","text":"<p>The interface between the execution infrastructure and the test suite is an Ansible inventory and variables files. Inventory and variable samples are provided. To run the tests, follow this procedure:</p> <ul> <li>Install dependencies and create a venv:</li> </ul> <pre><code>sudo dnf -y install python-devel\npython3 -m venv venv\nsource venv/bin/activate\npip install openstackclient osc_placement jmespath\nansible-galaxy collection install community.general\n</code></pre> <ul> <li>Run <code>make test-with-ceph</code> (the documented development environment   does include Ceph).</li> </ul> <p>If you are using Ceph-less environment, you should run <code>make   test-minimal</code>.</p>"},{"location":"contributing/tests/#making-patches-to-the-test-suite","title":"Making patches to the test suite","text":"<p>Please be aware of the following when changing the test suite:</p> <ul> <li>The test suite should follow the docs as much as possible.</li> </ul> <p>The purpose of the test suite is to verify what the user would run   if they were following the docs. We don't want to loosely rewrite   the docs into Ansible code following Ansible best practices. We want   to test the exact same bash commands/snippets that are written in   the docs. This often means that we should be using the <code>shell</code>   module and do a verbatim copy/paste from docs, instead of using the   best Ansible module for the task at hand.</p>"},{"location":"openstack/backend_services_deployment/","title":"Backend services deployment","text":"<p>The following instructions create OpenStackControlPlane CR with basic backend services deployed, and all the OpenStack services disabled. This will be the foundation of the podified control plane.</p> <p>In subsequent steps, we'll import the original databases and then add podified OpenStack control plane services.</p>"},{"location":"openstack/backend_services_deployment/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>The cloud which we want to adopt is up and running. It's on   OpenStack Wallaby release.</p> </li> <li> <p>The <code>openstack-operator</code> is deployed, but <code>OpenStackControlPlane</code> is   not deployed.</p> </li> </ul> <p>For developer/CI environments, the openstack operator can be deployed   by running <code>make openstack</code> inside   install_yamls   repo.</p> <p>For production environments, the deployment method will likely be   different.</p> <ul> <li>There are free PVs available to be claimed (for MariaDB and RabbitMQ).</li> </ul> <p>For developer/CI environments driven by install_yamls, make sure   you've run <code>make crc_storage</code>.</p>"},{"location":"openstack/backend_services_deployment/#variables","title":"Variables","text":"<ul> <li>Set the desired admin password for the podified deployment. This can   be the original deployment's admin password or something else.</li> </ul> <pre><code>ADMIN_PASSWORD=SomePassword\n</code></pre> <p>To use the existing OpenStack deployment password:   <pre><code>ADMIN_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' AdminPassword:' | awk -F ': ' '{ print $2; }')\n</code></pre></p> <ul> <li>Set service password variables to match the original deployment.   Database passwords can differ in podified environment, but   synchronizing the service account passwords is a required step.</li> </ul> <p>E.g. in developer environments with TripleO Standalone, the   passwords can be extracted like this:</p> <pre><code>CINDER_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' CinderPassword:' | awk -F ': ' '{ print $2; }')\nGLANCE_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' GlancePassword:' | awk -F ': ' '{ print $2; }')\nIRONIC_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' IronicPassword:' | awk -F ': ' '{ print $2; }')\nNEUTRON_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NeutronPassword:' | awk -F ': ' '{ print $2; }')\nNOVA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NovaPassword:' | awk -F ': ' '{ print $2; }')\nOCTAVIA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' OctaviaPassword:' | awk -F ': ' '{ print $2; }')\nPLACEMENT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' PlacementPassword:' | awk -F ': ' '{ print $2; }')\nHEAT_AUTH_ENCRYPTION_KEY=$(cat ~/tripleo-standalone-passwords.yaml | grep ' HeatAuthEncryptionKey:' | awk -F ': ' '{ print $2; }')\nHEAT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' HeatPassword:' | awk -F ': ' '{ print $2; }')\n</code></pre>"},{"location":"openstack/backend_services_deployment/#pre-checks","title":"Pre-checks","text":""},{"location":"openstack/backend_services_deployment/#procedure-backend-services-deployment","title":"Procedure - backend services deployment","text":"<ul> <li>Make sure you are using the OpenShift namespace where you want the   podified control plane deployed:</li> </ul> <pre><code>oc project openstack\n</code></pre> <ul> <li>Create OSP secret.</li> </ul> <p>The procedure for this will vary, but in developer/CI environments   we use install_yamls:</p> <pre><code># in install_yamls\nmake input\n</code></pre> <ul> <li>If the <code>$ADMIN_PASSWORD</code> is different than the already set password   in <code>osp-secret</code>, amend the <code>AdminPassword</code> key in the <code>osp-secret</code>   correspondingly:</li> </ul> <pre><code>oc set data secret/osp-secret \"AdminPassword=$ADMIN_PASSWORD\"\n</code></pre> <ul> <li>Set service account passwords in <code>osp-secret</code> to match the service   account passwords from the original deployment:</li> </ul> <pre><code>oc set data secret/osp-secret \"CinderPassword=$CINDER_PASSWORD\"\noc set data secret/osp-secret \"GlancePassword=$GLANCE_PASSWORD\"\noc set data secret/osp-secret \"IronicPassword=$IRONIC_PASSWORD\"\noc set data secret/osp-secret \"NeutronPassword=$NEUTRON_PASSWORD\"\noc set data secret/osp-secret \"NovaPassword=$NOVA_PASSWORD\"\noc set data secret/osp-secret \"OctaviaPassword=$OCTAVIA_PASSWORD\"\noc set data secret/osp-secret \"PlacementPassword=$PLACEMENT_PASSWORD\"\noc set data secret/osp-secret \"HeatPassword=$HEAT_PASSWORD\"\noc set data secret/osp-secret \"HeatAuthEncryptionKey=$HEAT_AUTH_ENCRYPTION_KEY\"\n</code></pre> <ul> <li>Deploy OpenStackControlPlane. Make sure to only enable DNS,   MariaDB, Memcached, and RabbitMQ services. All other services must   be disabled.</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n\n  cinder:\n    enabled: false\n    template:\n      cinderAPI: {}\n      cinderScheduler: {}\n      cinderBackup: {}\n      cinderVolumes: {}\n\n  dns:\n    template:\n      override:\n        service:\n          metadata:\n            annotations:\n              metallb.universe.tf/address-pool: ctlplane\n              metallb.universe.tf/allow-shared-ip: ctlplane\n              metallb.universe.tf/loadBalancerIPs: 192.168.122.80\n          spec:\n            type: LoadBalancer\n      options:\n      - key: server\n        values:\n        - 192.168.122.1\n      replicas: 1\n\n  glance:\n    enabled: false\n    template:\n      glanceAPIInternal: {}\n      glanceAPIExternal: {}\n\n  horizon:\n    enabled: false\n    template: {}\n\n  ironic:\n    enabled: false\n    template:\n      ironicConductors: []\n\n  keystone:\n    enabled: false\n    template: {}\n\n  manila:\n    enabled: false\n    template:\n      manilaAPI: {}\n      manilaScheduler: {}\n      manilaShares: {}\n\n  mariadb:\n    templates:\n      openstack:\n        storageRequest: 500M\n      openstack-cell1:\n        storageRequest: 500M\n\n  memcached:\n    enabled: true\n    templates:\n      memcached:\n        replicas: 1\n\n  neutron:\n    enabled: false\n    template: {}\n\n  nova:\n    enabled: false\n    template: {}\n\n  ovn:\n    enabled: false\n    template:\n      ovnController:\n        external-ids:\n          system-id: \"random\"\n          ovn-bridge: \"br-int\"\n          ovn-encap-type: \"geneve\"\n\n  placement:\n    enabled: false\n    template: {}\n\n  rabbitmq:\n    templates:\n      rabbitmq:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.85\n            spec:\n              type: LoadBalancer\n      rabbitmq-cell1:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.86\n            spec:\n              type: LoadBalancer\n\n  telemetry:\n    enabled: false\n    template: {}\nEOF\n</code></pre>"},{"location":"openstack/backend_services_deployment/#post-checks","title":"Post-checks","text":"<ul> <li>Check that MariaDB is running.</li> </ul> <pre><code>oc get pod mariadb-openstack -o jsonpath='{.status.phase}{\"\\n\"}'\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/","title":"Ceph backend configuration (if applicable)","text":"<p>If the original deployment uses a Ceph storage backend for any service (e.g. Glance, Cinder, Nova), the same backend must be used in the adopted deployment and CRs must be configured accordingly.</p>"},{"location":"openstack/ceph_backend_configuration/#prerequisites","title":"Prerequisites","text":"<ul> <li>The <code>OpenStackControlPlane</code> CR must already exist.</li> </ul>"},{"location":"openstack/ceph_backend_configuration/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>CEPH_SSH=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\"\nCEPH_KEY=$($CEPH_SSH \"cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0\")\nCEPH_CONF=$($CEPH_SSH \"cat /etc/ceph/ceph.conf | base64 -w 0\")\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/#procedure-ceph-backend-configuration","title":"Procedure - Ceph backend configuration","text":"<p>Create the <code>ceph-conf-files</code> secret, containing Ceph configuration:</p> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\ndata:\n  ceph.client.openstack.keyring: $CEPH_KEY\n  ceph.conf: $CEPH_CONF\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\ntype: Opaque\nEOF\n</code></pre> <p>The content of the file should look something like this:</p> <pre><code>---\napiVersion: v1\nkind: Secret\nmetadata:\n  name: ceph-conf-files\n  namespace: openstack\nstringData:\n  ceph.client.openstack.keyring: |\n    [client.openstack]\n        key = &lt;secret key&gt;\n        caps mgr = \"allow *\"\n        caps mon = \"profile rbd\"\n        caps osd = \"profile rbd pool=images\"\n  ceph.conf: |\n    [global]\n    fsid = 7a1719e8-9c59-49e2-ae2b-d7eb08c695d4\n    mon_host = 10.1.1.2,10.1.1.3,10.1.1.4\n</code></pre> <p>Configure <code>extraMounts</code> within the <code>OpenStackControlPlane</code> CR:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  extraMounts:\n    - name: v1\n      region: r1\n      extraVol:\n        - propagation:\n          - CinderVolume\n          - CinderBackup\n          - GlanceAPI\n          extraVolType: Ceph\n          volumes:\n          - name: ceph\n            projected:\n              sources:\n              - secret:\n                  name: ceph-conf-files\n          mounts:\n          - name: ceph\n            mountPath: \"/etc/ceph\"\n            readOnly: true\n'\n</code></pre>"},{"location":"openstack/ceph_backend_configuration/#getting-ceph-fsid","title":"Getting Ceph FSID","text":"<p>Configuring some OpenStack services to use Ceph backend may require the FSID value. You can fetch the value from the config like so:</p> <pre><code>CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data.\"ceph.conf\"' | base64 -d | grep fsid | sed -e 's/fsid = //')\n</code></pre>"},{"location":"openstack/cinder_adoption/","title":"Cinder adoption","text":"<p>Adopting a director deployed Cinder service into OpenStack may require some thought because it's not always a simple process.</p> <p>Usually the adoption process entails:</p> <ul> <li>Checking existing limitations.</li> <li>Considering the placement of the cinder services.</li> <li>Preparing the OpenShift nodes where volume and backup services will run.</li> <li>Crafting the manifest based on the existing <code>cinder.conf</code> file.</li> <li>Deploying Cinder.</li> <li>Validating the new deployment.</li> </ul> <p>This guide provides necessary knowledge to complete these steps in most situations, but it still requires knowledge on how OpenStack services work and the structure of a Cinder configuration file.</p>"},{"location":"openstack/cinder_adoption/#limitations","title":"Limitations","text":"<p>There are currently some limitations that are worth highlighting; some are related to this guideline while some to the operator:</p> <ul> <li> <p>There is no global <code>nodeSelector</code> for all cinder volumes, so it needs to be specified per backend.  This may change in the future.</p> </li> <li> <p>There is no global <code>customServiceConfig</code> or <code>customServiceConfigSecrets</code> for all cinder volumes, so it needs to be specified per backend.  This may change in the future.</p> </li> <li> <p>Adoption of LVM backends, where the volume data is stored in the compute nodes, is not currently being documented in this process. It may get documented in the future.</p> </li> <li> <p>Support for Cinder backends that require kernel modules not included in RHEL has not been tested in Operator deployed OpenStack so it is not documented in this guide.</p> </li> <li> <p>Adoption of DCN/Edge deployment is not currently described in this guide.</p> </li> </ul>"},{"location":"openstack/cinder_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>Previous Adoption steps completed. Notably, cinder service must have been stopped and the service databases must already be imported into the podified MariaDB.</p> </li> <li> <p>Storage network has been properly configured on the OpenShift cluster.</p> </li> </ul>"},{"location":"openstack/cinder_adoption/#variables","title":"Variables","text":"<p>No new environmental variables need to be defined, though we use the <code>CONTROLLER1_SSH</code> that was defined in a previous step for the pre-checks.</p>"},{"location":"openstack/cinder_adoption/#pre-checks","title":"Pre-checks","text":"<p>We are going to need the contents of <code>cinder.conf</code>, so we may want to download it to have it locally accessible:</p> <pre><code>$CONTROLLER1_SSH cat /var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf &gt; cinder.conf\n</code></pre>"},{"location":"openstack/cinder_adoption/#prepare-openshift","title":"Prepare OpenShift","text":"<p>As explained the planning section before deploying OpenStack in OpenShift we need to ensure that the networks are ready, that we have decided the node selection, and also make sure any necessary changes to the OpenShift nodes have been made.  For Cinder volume and backup services all these 3 must be carefully considered.</p>"},{"location":"openstack/cinder_adoption/#node-selection","title":"Node Selection","text":"<p>We may need, or want, to restrict the OpenShift nodes where cinder volume and backup services can run.</p> <p>The best example of when we need to do node selection for a specific cinder service in when we deploy Cinder with the LVM driver. In that scenario the LVM data where the volumes are stored only exists in a specific host, so we need to pin the cinder-volume service to that specific OpenShift node.  Running the service on any other OpenShift node would not work.  Since <code>nodeSelector</code> only works on labels we cannot use the OpenShift host node name to restrict the LVM backend and we'll need to identify it using a unique label, an existing or new one:</p> <pre><code>$ oc label nodes worker0 lvm=cinder-volumes\n</code></pre> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  cinder:\n    enabled: true\n    template:\n      cinderVolumes:\n        lvm-iscsi:\n          nodeSelector:\n            lvm: cinder-volumes\n&lt; . . . &gt;\n</code></pre> <p>As mentioned in the Node Selector guide, an example where we need to use labels is when using FC storage and we don't have HBA cards in all our OpenShift nodes. In this scenario we would need to restrict all the cinder volume backends (not only the FC one) as well as the backup services.</p> <p>Depending on the cinder backends, their configuration, and the usage of Cinder, we can have network intensive cinder volume services with lots of I/O as well as cinder backup services that are not only network intensive but also memory and CPU intensive.  This may be a concern for the OpenShift human operators, and they may want to use the <code>nodeSelector</code> to prevent these service from interfering with their other OpenShift workloads</p> <p>Please make sure to read the Nodes Selector guide before continuing, as we'll be referring to some of the concepts explained there in the following sections.</p> <p>When selecting the nodes where cinder volume is going to run please remember that cinder-volume may also use local storage when downloading a glance image for the create volume from image operation, and it can require a considerable amount of space when having concurrent operations and not using cinder volume cache.</p> <p>If we don't have nodes with enough local disk space for the temporary images we can use a remote NFS location for the images. This is something that we had to manually setup in Director deployments, but with operators we can easily do it automatically using the extra volumes feature ()<code>extraMounts</code>.</p>"},{"location":"openstack/cinder_adoption/#transport-protocols","title":"Transport protocols","text":"<p>Due to the specifics of the storage transport protocols some changes may be required on the OpenShift side, and although this is something that must be documented by the Vendor here wer are going to provide some generic instructions that can serve as a guide for the different transport protocols.</p> <p>Check the backend sections in our <code>cinder.conf</code> file that are listed in the <code>enabled_backends</code> configuration option to figure out the transport storage protocol used by the backend.</p> <p>Depending on the backend we can find the transport protocol:</p> <ul> <li> <p>Looking at the <code>volume_driver</code> configuration option, as it may contain the   protocol itself: RBD, iSCSI, FC...</p> </li> <li> <p>Looking at the <code>target_protocol</code> configuration option</p> </li> </ul> <p>Warning: Any time a <code>MachineConfig</code> is used to make changes to OpenShift nodes the node will reboot!!  Act accordingly.</p>"},{"location":"openstack/cinder_adoption/#nfs","title":"NFS","text":"<p>There's nothing to do for NFS. OpenShift can connect to NFS backends without any additional changes.</p>"},{"location":"openstack/cinder_adoption/#rbdceph","title":"RBD/Ceph","text":"<p>There's nothing to do for RBD/Ceph in terms of preparing the nodes, OpenShift can connect to Ceph backends without any additional changes. Credentials and configuration files will need to be provided to the services though.</p>"},{"location":"openstack/cinder_adoption/#iscsi","title":"iSCSI","text":"<p>Connecting to iSCSI volumes requires that the iSCSI initiator is running on the OpenShift hosts hosts where volume and backup services are going to run, because the Linux Open iSCSI initiator doesn't currently support network namespaces, so we must only run 1 instance of the service for the normal OpenShift usage, plus the OpenShift CSI plugins, plus the OpenStack services.</p> <p>If we are not already running <code>iscsid</code> on the OpenShift nodes then we'll need to apply a <code>MachineConfig</code> similar to this one:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-enable-iscsid\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    systemd:\n      units:\n      - enabled: true\n        name: iscsid.service\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>For production deployments using iSCSI volumes we always recommend setting up multipathing, please look at the multipathing section to see how to configure it.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for iSCSI.</p>"},{"location":"openstack/cinder_adoption/#fc","title":"FC","text":"<p>There's nothing to do for FC volumes to work, but the cinder volume and cinder backup services need to run in an OpenShift host that has HBAs, so if there are nodes that don't have HBAs then we'll need to use labels to restrict where these services can run, as mentioned in the node labels section.</p> <p>This also means that for virtualized OpenShift clusters using FC we'll need to expose the host's HBAs inside the VM.</p> <p>For production deployments using FC volumes we always recommend setting up multipathing, please look at the multipathing section to see how to configure it.</p>"},{"location":"openstack/cinder_adoption/#nvme-of","title":"NVMe-oF","text":"<p>Connecting to NVMe-oF volumes requires that the nvme kernel modules are loaded on the OpenShift hosts.</p> <p>If we are not already loading the <code>nvme-fabrics</code> module on the OpenShift nodes where volume and backup services are going to run then we'll need to apply a <code>MachineConfig</code> similar to this one:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-load-nvme-fabrics\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    storage:\n      files:\n        - path: /etc/modules-load.d/nvme_fabrics.conf\n          overwrite: false\n          # Mode must be decimal, this is 0644\n          mode: 420\n          user:\n            name: root\n          group:\n            name: root\n          contents:\n            # Source can be a http, https, tftp, s3, gs, or data as defined in rfc2397.\n            # This is the rfc2397 text/plain string format\n            source: data:,nvme-fabrics\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>We are only loading the <code>nvme-fabrics</code> module because it takes care of loading the transport specific modules (tcp, rdma, fc) as needed.</p> <p>For production deployments using NVMe-oF volumes we always recommend using multipathing. For NVMe-oF volumes OpenStack uses native multipathing, called ANA.</p> <p>Once the OpenShift nodes have rebooted and are loading the <code>nvme-fabrics</code> module we can confirm that the Operating System is configured and supports ANA by checking on the host:</p> <pre><code>cat /sys/module/nvme_core/parameters/multipath\n</code></pre> <p>Attention: ANA doesn't use the Linux Multipathing Device Mapper, but the *current OpenStack code requires <code>multipathd</code> on compute nodes to be running for Nova to be able to use multipathing, so please remember to follow the multipathing part for compute nodes on the multipathing section.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for NVMe-oF.</p>"},{"location":"openstack/cinder_adoption/#multipathing","title":"Multipathing","text":"<p>For iSCSI and FC protocols we always recommend using multipathing, which has 4 parts:</p> <ul> <li>Prepare the OpenShift hosts</li> <li>Configure the Cinder services</li> <li>Prepare the Nova computes</li> <li>Configure the Nova service</li> </ul> <p>To prepare the OpenShift hosts we need to ensure that the Linux Multipath Device Mapper is configured and running on the OpenShift hosts, and we do that using <code>MachineConfig</code> like this one:</p> <pre><code># Includes the /etc/multipathd.conf contents and the systemd unit changes\napiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: worker\n    service: cinder\n  name: 99-master-cinder-enable-multipathd\nspec:\n  config:\n    ignition:\n      version: 3.2.0\n    storage:\n      files:\n        - path: /etc/multipath.conf\n          overwrite: false\n          # Mode must be decimal, this is 0600\n          mode: 384\n          user:\n            name: root\n          group:\n            name: root\n          contents:\n            # Source can be a http, https, tftp, s3, gs, or data as defined in rfc2397.\n            # This is the rfc2397 text/plain string format\n            source: data:,defaults%20%7B%0A%20%20user_friendly_names%20no%0A%20%20recheck_wwid%20yes%0A%20%20skip_kpartx%20yes%0A%20%20find_multipaths%20yes%0A%7D%0A%0Ablacklist%20%7B%0A%7D\n    systemd:\n      units:\n      - enabled: true\n        name: multipathd.service\n</code></pre> <p>Remember that if we are using labels to restrict the nodes where cinder services are running we'll need to use a <code>MachineConfigPool</code> as described in the nodes selector guide to limit the effects of the <code>MachineConfig</code> to only the nodes were our services may run.</p> <p>If we are using a toy single node deployment to test the process we may need to replace <code>worker</code> with <code>master</code> in the <code>MachineConfig</code>.</p> <p>To configure the cinder services to use multipathing we need to enable the <code>use_multipath_for_image_xfer</code> configuration option in all the backend sections and in the <code>[DEFAULT]</code> section for the backup service, but in Podified deployments we don't need to worry about it, because that's the default. So as long as we don't override it setting <code>use_multipath_for_image_xfer = false</code> then multipathing will work as long as the service is running on the OpenShift host.</p> <p>TODO: Add, or at least mention, the Nova eDPM side for Multipathing once it's implemented.</p>"},{"location":"openstack/cinder_adoption/#configurations","title":"Configurations","text":"<p>As described in the planning Cinder is configured using configuration snippets instead of using obscure configuration parameters defined by the installer.</p> <p>The recommended way to deploy Cinder volume backends has changed to remove old limitations, add flexibility, and improve operations in general.</p> <p>When deploying with Director we used to run a single Cinder volume service with all our backends (each backend would run on its own process), and even though that way of deploying is still supported, we don't recommend it. We recommend using a volume service per backend since it's a superior deployment model.</p> <p>So for an LVM and a Ceph backend we would have 2 entries in <code>cinderVolume</code> and, as mentioned in the limitations section, we cannot set global defaults for all volume services, so we would have to define it for each of them, like this:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    enabled: true\n    template:\n      cinderVolume:\n        lvm:\n          customServiceConfig: |\n            [DEFAULT]\n            debug = True\n            [lvm]\n&lt; . . . &gt;\n        ceph:\n          customServiceConfig: |\n            [DEFAULT]\n            debug = True\n            [ceph]\n&lt; . . . &gt;\n</code></pre> <p>Reminder that for volume backends that have sensitive information using <code>Secret</code> and the <code>customServiceConfigSecrets</code> key is the recommended way to go.</p>"},{"location":"openstack/cinder_adoption/#prepare-the-configuration","title":"Prepare the configuration","text":"<p>For adoption instead of using a whole deployment manifest we'll use a targeted patch, like we did with other services, and in this patch we will enable the different cinder services with their specific configurations.</p> <p>WARNING: Check that all configuration options are still valid for the new OpenStack version, since configuration options may have been deprecated, removed, or added. This applies to both backend driver specific configuration options and other generic options.</p> <p>There are 2 ways to prepare a cinder configuration for adoption, tailor-making it or doing it quick and dirty. There is no difference in how Cinder will operate with both methods, so we are free to chose, though we recommend tailor-making it whenever possible.</p> <p>The high level explanation of the tailor-made approach is:</p> <ol> <li> <p>Determine what part of the configuration is generic for all the cinder services and remove anything that would change when deployed in OpenShift, like the <code>connection</code> in the <code>[dabase]</code> section, the <code>transport_url</code> and <code>log_dir</code> in <code>[DEFAULT]</code>, the whole <code>[coordination]</code> section.  This configuration goes into the <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>) at the <code>cinder: template:</code> level.</p> </li> <li> <p>Determine if there's any scheduler specific configuration and add it to the <code>customServiceConfig</code> section in <code>cinder: template: cinderScheduler</code>.</p> </li> <li> <p>Determine if there's any API specific configuration and add it to the <code>customServiceConfig</code> section in <code>cinder: template: cinderAPI</code>.</p> </li> <li> <p>If we have cinder backup deployed, then we'll get the cinder backup relevant configuration options and add them to <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>) at the <code>cinder: template: cinderBackup:</code> level. We should remove the <code>host</code> configuration in the <code>[DEFAULT]</code> section to facilitate supporting multiple replicas in the future.</p> </li> <li> <p>Determine the individual volume backend configuration for each of the drivers. The configuration will not only be the specific driver section, it should also include the <code>[backend_defaults]</code> section and FC zoning sections is they are being used, because the cinder operator doesn't support a <code>customServiceConfig</code> section global for all volume services.  Each backend would have its own section under <code>cinder: template: cinderVolumes</code> and the configuration would go in <code>customServiceConfig</code> (or a <code>Secret</code> and then used in <code>customServiceConfigSecrets</code>).</p> </li> <li> <p>Check if any of the cinder volume drivers being used requires a custom vendor image. If they do, find the location of the image in the vendor's instruction available in the w OpenStack Cinder ecosystem page and add it under the specific's driver section using the <code>containerImage</code> key. For example, if we had a Pure Storage array and the driver was already certified for OSP18, then we would have something like this:</p> </li> </ol> <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderVolume:\n        pure:\n          containerImage: registry.connect.redhat.com/purestorage/openstack-cinder-volume-pure-rhosp-18-0'\n          customServiceConfigSecrets:\n            - openstack-cinder-pure-cfg\n&lt; . . . &gt;\n</code></pre> <ol> <li>External files: Cinder services sometimes use external files, for example for a custom policy, or to store credentials, or SSL CA bundles to connect to a storage array, and we need to make those files available to the right containers. To achieve this we'll use <code>Secrets</code> or <code>ConfigMap</code> to store the information in OpenShift and then the <code>extraMounts</code> key. For example, for the Ceph credentials stored in a <code>Secret</code> called <code>ceph-conf-files</code> we would patch the top level <code>extraMounts</code> in <code>OpenstackControlPlane</code>:</li> </ol> <p><pre><code>spec:\n  extraMounts:\n  - extraVol:\n    - extraVolType: Ceph\n      mounts:\n      - mountPath: /etc/ceph\n        name: ceph\n        readOnly: true\n      propagation:\n      - CinderVolume\n      - CinderBackup\n      - Glance\n      volumes:\n      - name: ceph\n        projected:\n          sources:\n          - secret:\n              name: ceph-conf-files\n</code></pre>    But for a service specific one, like the API policy, we would do it directly    on the service itself, in this example we include the cinder API    configuration that references the policy we are adding from a <code>ConfigMap</code>    called <code>my-cinder-conf</code> that has a key <code>policy</code> with the contents of the    policy:    <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderAPI:\n        customServiceConfig: |\n           [oslo_policy]\n           policy_file=/etc/cinder/api/policy.yaml\n      extraMounts:\n      - extraVol:\n        - extraVolType: Ceph\n          mounts:\n          - mountPath: /etc/cinder/api\n            name: policy\n            readOnly: true\n          propagation:\n          - CinderAPI\n          volumes:\n          - name: policy\n            projected:\n              sources:\n              - configMap:\n                  name: my-cinder-conf\n                  items:\n                    - key: policy\n                      path: policy.yaml\n</code></pre></p> <p>The quick and dirty process is more straightforward:</p> <ol> <li> <p>Create an agnostic configuration file removing any specifics from the old deployment's <code>cinder.conf</code> file, like the <code>connection</code> in the <code>[dabase]</code> section, the <code>transport_url</code> and <code>log_dir</code> in <code>[DEFAULT]</code>, the whole <code>[coordination]</code> section, etc..</p> </li> <li> <p>Assuming the configuration has sensitive information, drop the modified contents of the whole file into a <code>Secret</code>.</p> </li> <li> <p>Reference this secret in all the services, creating a cinder volumes section for each backend and just adding the respective <code>enabled_backends</code> option.</p> </li> <li> <p>Add external files as mentioned in the last bullet of the tailor-made configuration explanation.</p> </li> </ol> <p>Example of what the quick and dirty configuration patch would look like:</p> <pre><code>spec:\n  cinder:\n    enabled: true\n    template:\n      cinderAPI:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderScheduler:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderBackup:\n        customServiceConfigSecrets:\n          - cinder-conf\n      cinderVolume:\n        lvm1:\n          customServiceConfig: |\n            [DEFAULT]\n            enabled_backends = lvm1\n          customServiceConfigSecrets:\n            - cinder-conf\n        lvm2:\n          customServiceConfig: |\n            [DEFAULT]\n            enabled_backends = lvm2\n          customServiceConfigSecrets:\n            - cinder-conf\n</code></pre>"},{"location":"openstack/cinder_adoption/#configuration-generation-helper-tool","title":"Configuration generation helper tool","text":"<p>Creating the right Cinder configuration files to deploy using Operators may sometimes be a complicated experience, especially the first times, so we have a helper tool that can create a draft of the files from a <code>cinder.conf</code> file.</p> <p>This tool is not meant to be a automation tool, it's mostly to help us get the gist of it, maybe point out some potential pitfalls and reminders.</p> <p>Attention: The tools requires <code>PyYAML</code> Python package to be installed (<code>pip install PyYAML</code>).</p> <p>This cinder-cfg.py script defaults to reading the <code>cinder.conf</code> file from the current directory (unless <code>--config</code> option is used) and outputs files to the current directory (unless <code>--out-dir</code> option is used).</p> <p>In the output directory we'll always get a <code>cinder.patch</code> file with the Cinder specific configuration patch to apply to the <code>OpenStackControlPlane</code> CR but we may also get an additional file called <code>cinder-prereq.yaml</code> file with some <code>Secrets</code> and <code>MachineConfigs</code>.</p> <p>Example of an invocation setting input and output explicitly to the defaults for a Ceph backend:</p> <pre><code>$ python cinder-cfg.py --config cinder.conf --out-dir ./\nWARNING:root:Cinder is configured to use ['/etc/cinder/policy.yaml'] as policy file, please ensure this file is available for the podified cinder services using \"extraMounts\" or remove the option.\n\nWARNING:root:Deployment uses Ceph, so make sure the Ceph credentials and configuration are present in OpenShift as a asecret and then use the extra volumes to make them available in all the services that would need them.\n\nWARNING:root:You were using user ['nova'] to talk to Nova, but in podified we prefer using the service keystone username, in this case ['cinder']. Dropping that configuration.\n\nWARNING:root:ALWAYS REVIEW RESULTS, OUTPUT IS JUST A ROUGH DRAFT!!\n\nOutput written at ./: cinder.patch\n</code></pre> <p>The script outputs some warnings to let us know things we may need to do manually -adding the custom policy, provide the ceph configuration files- and also let us know a change in how the <code>service_user</code> has been removed.</p> <p>A different example when using multiple backends, one of them being a 3PAR FC could be:</p> <pre><code>$ python cinder-cfg.py --config cinder.conf --out-dir ./\nWARNING:root:Cinder is configured to use ['/etc/cinder/policy.yaml'] as policy file, please ensure this file is available for the podified cinder services using \"extraMounts\" or remove the option.\n\nERROR:root:Backend hpe_fc requires a vendor container image, but there is no certified image available yet. Patch will use the last known image for reference, but IT WILL NOT WORK\n\nWARNING:root:Deployment uses Ceph, so make sure the Ceph credentials and configuration are present in OpenShift as a asecret and then use the extra volumes to make them available in all the services that would need them.\n\nWARNING:root:You were using user ['nova'] to talk to Nova, but in podified we prefer using the service keystone username, in this case ['cinder']. Dropping that configuration.\n\nWARNING:root:Configuration is using FC, please ensure all your OpenShift nodes have HBAs or use labels to ensure that Volume and Backup services are scheduled on nodes with HBAs.\n\nWARNING:root:ALWAYS REVIEW RESULTS, OUTPUT IS JUST A ROUGH DRAFT!!\n\nOutput written at ./: cinder.patch, cinder-prereq.yaml\n</code></pre> <p>In this case we can see that there are additional messages, so let's quickly go over them:</p> <ul> <li>There's one message mentioning how this backend driver needs external vendor dependencies so the standard container image will not work. Unfortunately this image is still not available, so an older image is used in the output patch file for reference. We can then replace this image with one we build ourselves or with a Red Hat official one once the image is available. In this case we can see in our <code>cinder.patch</code> file:   <pre><code>      cinderVolumes:\n      hpe-fc:\n        containerImage: registry.connect.redhat.com/hpe3parcinder/openstack-cinder-volume-hpe3parcinder17-0\n</code></pre></li> <li> <p>The FC message reminds us that this transport protocol requires specific HBA cards to be present on the nodes where cinder services are running.</p> </li> <li> <p>In this case we also see that it has created the <code>cinder-prereq.yaml</code> file and if we look into it we'll see there is one <code>MachineConfig</code> and one <code>Secret</code>. The <code>MachineConfig</code> is called <code>99-master-cinder-enable-multipathd</code> and like the name suggests enables multipathing on all the OCP worker nodes. The <code>Secret</code> is called <code>openstackcinder-volumes-hpe_fc</code> and contains the 3PAR backend configuration because it has sensitive information (credentials), and in the <code>cinder.patch</code> file we'll see that it uses this configuration:   <pre><code>   cinderVolumes:\n      hpe-fc:\n        customServiceConfigSecrets:\n        - openstackcinder-volumes-hpe_fc\n</code></pre></p> </li> </ul>"},{"location":"openstack/cinder_adoption/#procedure-cinder-adoption","title":"Procedure - Cinder adoption","text":"<p>Assuming we have already stopped cinder services, prepared the OpenShift nodes, deployed the OpenStack operators and a bare OpenStack manifest, and migrated the database, and prepared the patch manifest with the Cinder service configuration, all that's left is to apply the patch and wait for the operator to apply the changes and deploy the Cinder services.</p> <p>Our recommendation is to write the patch manifest into a file, for example <code>cinder.patch</code> and then apply it with something like:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch-file=cinder.patch\n</code></pre> <p>For example, for the RBD deployment from the Development Guide the <code>cinder.patch</code> would look like this:</p> <pre><code>spec:\n  extraMounts:\n  - extraVol:\n    - extraVolType: Ceph\n      mounts:\n      - mountPath: /etc/ceph\n        name: ceph\n        readOnly: true\n      propagation:\n      - CinderVolume\n      - CinderBackup\n      - Glance\n      volumes:\n      - name: ceph\n        projected:\n          sources:\n          - secret:\n              name: ceph-conf-files\n  cinder:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      secret: osp-secret\n      cinderAPI:\n        override:\n          service:\n            internal:\n              metadata:\n                annotations:\n                  metallb.universe.tf/address-pool: internalapi\n                  metallb.universe.tf/allow-shared-ip: internalapi\n                  metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n              spec:\n                type: LoadBalancer\n        replicas: 1\n        customServiceConfig: |\n          [DEFAULT]\n          default_volume_type=tripleo\n      cinderScheduler:\n        replicas: 1\n      cinderBackup:\n        networkAttachments:\n        - storage\n        replicas: 1\n        customServiceConfig: |\n          [DEFAULT]\n          backup_driver=cinder.backup.drivers.ceph.CephBackupDriver\n          backup_ceph_conf=/etc/ceph/ceph.conf\n          backup_ceph_user=openstack\n          backup_ceph_pool=backups\n      cinderVolumes:\n        ceph:\n          networkAttachments:\n          - storage\n          replicas: 1\n          customServiceConfig: |\n            [tripleo_ceph]\n            backend_host=hostgroup\n            volume_backend_name=tripleo_ceph\n            volume_driver=cinder.volume.drivers.rbd.RBDDriver\n            rbd_ceph_conf=/etc/ceph/ceph.conf\n            rbd_user=openstack\n            rbd_pool=volumes\n            rbd_flatten_volume_from_snapshot=False\n            report_discard_supported=True\n</code></pre> <p>Once the services have been deployed we'll need to clean up the old scheduler and backup services which will appear as being down while we have others that appear as being up:</p> <pre><code>openstack volume service list\n\n+------------------+------------------------+------+---------+-------+----------------------------+\n| Binary           | Host                   | Zone | Status  | State | Updated At                 |\n+------------------+------------------------+------+---------+-------+----------------------------+\n| cinder-backup    | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:59.000000 |\n| cinder-scheduler | standalone.localdomain | nova | enabled | down  | 2023-06-28T11:00:29.000000 |\n| cinder-volume    | hostgroup@tripleo_ceph | nova | enabled | up    | 2023-06-28T17:00:03.000000 |\n| cinder-scheduler | cinder-scheduler-0     | nova | enabled | up    | 2023-06-28T17:00:02.000000 |\n| cinder-backup    | cinder-backup-0        | nova | enabled | up    | 2023-06-28T17:00:01.000000 |\n+------------------+------------------------+------+---------+-------+----------------------------+\n</code></pre> <p>In this case we need to remove services for hosts <code>standalone.localdomain</code></p> <pre><code>oc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-backup standalone.localdomain\noc exec -it cinder-scheduler-0 -- cinder-manage service remove cinder-scheduler standalone.localdomain\n</code></pre> <p>The reason why we haven't preserved the name of the backup service is because we have taken the opportunity to change its configuration to support Active-Active, even though we are not doing so right now because we have 1 replica.</p> <p>Now that we have the Cinder services running we know that the DB schema migration has been completed and we can proceed to apply the DB data migrations. While it is not necessary to run these data migrations at this precise moment, because we can just run them right before the next upgrade, we consider that for adoption it's best to run them now to make sure there are no issues before running production workloads on the deployment.</p> <p>The command to run the DB data migrations is:</p> <pre><code>oc exec -it cinder-scheduler-0 -- cinder-manage db online_data_migrations\n</code></pre>"},{"location":"openstack/cinder_adoption/#post-checks","title":"Post-checks","text":"<p>Before we can run any checks we need to set the right cloud configuration for the <code>openstack</code> command to be able to connect to our OpenShift control plane.</p> <p>Just like we did in the KeyStone adoption step we ensure we have the <code>openstack</code> alias defined:</p> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n</code></pre> <p>Now we can run a set of tests to confirm that the deployment is there using our old database contents:</p> <ul> <li>See that Cinder endpoints are defined and pointing to the podified   FQDNs:</li> </ul> <pre><code>openstack endpoint list --service cinderv3\n</code></pre> <ul> <li>Check that the cinder services are running and up. The API won't show but if   you get a response you know it's up as well:</li> </ul> <pre><code>openstack volume service list\n</code></pre> <ul> <li>Check that our old volume types, volumes, snapshots, and backups are there:</li> </ul> <pre><code>openstack volume type list\nopenstack volume list\nopenstack volume snapshot list\nopenstack volume backup list\n</code></pre> <p>To confirm that everything not only looks good but it's also properly working we recommend doing some basic operations:</p> <ul> <li> <p>Create a volume from an image to check that the connection to glance is   working.   <pre><code>openstack volume create --image cirros --bootable --size 1 disk_new\n</code></pre></p> </li> <li> <p>Backup the old attached volume to a new backup. Example:   <pre><code>openstack --os-volume-api-version 3.47 volume create --backup backup restored\n</code></pre></p> </li> </ul> <p>We don't boot a nova instance using the new volume from image or try to detach the old volume because nova and cinder are still not connected.</p>"},{"location":"openstack/edpm_adoption/","title":"EDPM adoption","text":""},{"location":"openstack/edpm_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed.</li> </ul>"},{"location":"openstack/edpm_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/edpm_adoption/#pre-checks","title":"Pre-checks","text":"<ul> <li>Make sure the IPAM is configured</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: network.openstack.org/v1beta1\nkind: NetConfig\nmetadata:\n  name: netconfig\nspec:\n  networks:\n  - name: CtlPlane\n    dnsDomain: ctlplane.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 192.168.122.120\n        start: 192.168.122.100\n      - end: 192.168.122.200\n        start: 192.168.122.150\n      cidr: 192.168.122.0/24\n      gateway: 192.168.122.1\n  - name: InternalApi\n    dnsDomain: internalapi.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.17.0.250\n        start: 172.17.0.100\n      cidr: 172.17.0.0/24\n      vlan: 20\n  - name: External\n    dnsDomain: external.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 10.0.0.250\n        start: 10.0.0.100\n      cidr: 10.0.0.0/24\n      gateway: 10.0.0.1\n  - name: Storage\n    dnsDomain: storage.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.18.0.250\n        start: 172.18.0.100\n      cidr: 172.18.0.0/24\n      vlan: 21\n  - name: StorageMgmt\n    dnsDomain: storagemgmt.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.20.0.250\n        start: 172.20.0.100\n      cidr: 172.20.0.0/24\n      vlan: 23\n  - name: Tenant\n    dnsDomain: tenant.example.com\n    subnets:\n    - name: subnet1\n      allocationRanges:\n      - end: 172.19.0.250\n        start: 172.19.0.100\n      cidr: 172.19.0.0/24\n      vlan: 22\nEOF\n</code></pre>"},{"location":"openstack/edpm_adoption/#procedure-edpm-adoption","title":"Procedure - EDPM adoption","text":"<ul> <li>Create a ssh authentication secret for the EDPM nodes:</li> </ul> <p><pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\nkind: Secret\nmetadata:\n    name: dataplane-adoption-secret\n    namespace: openstack\ndata:\n    ssh-privatekey: |\n$(cat ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa | base64 | sed 's/^/        /')\nEOF\n</code></pre> * Create the Nova Metadata secret (Workaround while nova isn't adopted yet):</p> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: v1\nkind: Secret\nmetadata:\n    name: nova-metadata-neutron-config\ndata:\n    05-nova-metadata.conf: |\n$(echo \"[DEFAULT]\\nnova_metadata_host = 1.2.3.4\\nnova_metadata_port = 8775\\nnova_metadata_protocol = http\\nmetadata_proxy_shared_secret = 1234567842\\n\" | base64 | sed 's/^/        /')\nEOF\n</code></pre> <ul> <li>Stop the nova services.</li> </ul> <pre><code># Update the services list to be stopped\n\nServicesToStop=(\"tripleo_nova_api_cron.service\"\n                \"tripleo_nova_api.service\"\n                \"tripleo_nova_compute.service\"\n                \"tripleo_nova_conductor.service\"\n                \"tripleo_nova_libvirt.target\"\n                \"tripleo_nova_metadata.service\"\n                \"tripleo_nova_migration_target.service\"\n                \"tripleo_nova_scheduler.service\"\n                \"tripleo_nova_virtlogd_wrapper.service\"\n                \"tripleo_nova_virtnodedevd.service\"\n                \"tripleo_nova_virtproxyd.service\"\n                \"tripleo_nova_virtqemud.service\"\n                \"tripleo_nova_virtsecretd.service\"\n                \"tripleo_nova_virtstoraged.service\"\n                \"tripleo_nova_vnc_proxy.service\")\n\necho \"Stopping nova services\"\n\nfor service in ${ServicesToStop[*]}; do\n    echo \"Stopping the $service in each controller node\"\n    $CONTROLLER1_SSH sudo systemctl stop $service\n    $CONTROLLER2_SSH sudo systemctl stop $service\n    $CONTROLLER3_SSH sudo systemctl stop $service\ndone\n</code></pre> <ul> <li>Deploy OpenStackDataPlaneNodeSet:</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: dataplane.openstack.org/v1beta1\nkind: OpenStackDataPlaneNodeSet\nmetadata:\n  name: openstack\nspec:\n  networkAttachments:\n      - ctlplane\n  preProvisioned: true\n  services:\n    - download-cache\n    - configure-network\n    - validate-network\n    - install-os\n    - configure-os\n    - run-os\n    - ovn\n  env:\n    - name: ANSIBLE_CALLBACKS_ENABLED\n      value: \"profile_tasks\"\n    - name: ANSIBLE_FORCE_COLOR\n      value: \"True\"\n    - name: ANSIBLE_ENABLE_TASK_DEBUGGER\n      value: \"True\"\n  nodes:\n    standalone:\n      hostName: standalone\n      ansible:\n        ansibleHost: 192.168.122.100\n      networks:\n      - defaultRoute: true\n        fixedIP: 192.168.122.100\n        name: CtlPlane\n        subnetName: subnet1\n      - name: InternalApi\n        subnetName: subnet1\n      - name: Storage\n        subnetName: subnet1\n      - name: Tenant\n        subnetName: subnet1\n  nodeTemplate:\n    ansibleSSHPrivateKeySecret: dataplane-adoption-secret\n    managementNetwork: ctlplane\n    ansible:\n      ansibleUser: root\n      ansiblePort: 22\n      ansibleVars:\n        service_net_map:\n          nova_api_network: internal_api\n          nova_libvirt_network: internal_api\n\n        # edpm_network_config\n        # Default nic config template for a EDPM compute node\n        # These vars are edpm_network_config role vars\n        edpm_network_config_override: \"\"\n        edpm_network_config_template: |\n           ---\n           {% set mtu_list = [ctlplane_mtu] %}\n           {% for network in role_networks %}\n           {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}\n           {%- endfor %}\n           {% set min_viable_mtu = mtu_list | max %}\n           network_config:\n           - type: ovs_bridge\n             name: {{ neutron_physical_bridge_name }}\n             mtu: {{ min_viable_mtu }}\n             use_dhcp: false\n             dns_servers: {{ ctlplane_dns_nameservers }}\n             domain: {{ dns_search_domains }}\n             addresses:\n             - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}\n             routes: {{ ctlplane_host_routes }}\n             members:\n             - type: interface\n               name: nic1\n               mtu: {{ min_viable_mtu }}\n               # force the MAC address of the bridge to this interface\n               primary: true\n           {% for network in role_networks %}\n             - type: vlan\n               mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}\n               vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}\n               addresses:\n               - ip_netmask:\n                   {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}\n               routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}\n           {% endfor %}\n\n        edpm_network_config_hide_sensitive_logs: false\n        #\n        # These vars are for the network config templates themselves and are\n        # considered EDPM network defaults.\n        neutron_physical_bridge_name: br-ctlplane\n        neutron_public_interface_name: eth0\n        role_networks:\n        - InternalApi\n        - Storage\n        - Tenant\n        networks_lower:\n          External: external\n          InternalApi: internal_api\n          Storage: storage\n          Tenant: tenant\n\n        # edpm_nodes_validation\n        edpm_nodes_validation_validate_controllers_icmp: false\n        edpm_nodes_validation_validate_gateway_icmp: false\n\n        edpm_chrony_ntp_servers:\n        - clock.redhat.com\n        - clock2.redhat.com\n\n        edpm_ovn_controller_agent_image: quay.io/podified-antelope-centos9/openstack-ovn-controller:current-podified\n        edpm_iscsid_image: quay.io/podified-antelope-centos9/openstack-iscsid:current-podified\n        edpm_logrotate_crond_image: quay.io/podified-antelope-centos9/openstack-cron:current-podified\n        edpm_nova_compute_container_image: quay.io/podified-antelope-centos9/openstack-nova-compute:current-podified\n        edpm_nova_libvirt_container_image: quay.io/podified-antelope-centos9/openstack-nova-libvirt:current-podified\n        edpm_ovn_metadata_agent_image: quay.io/podified-antelope-centos9/openstack-neutron-metadata-agent-ovn:current-podified\n\n        gather_facts: false\n        enable_debug: false\n        # edpm firewall, change the allowed CIDR if needed\n        edpm_sshd_configure_firewall: true\n        edpm_sshd_allowed_ranges: ['192.168.122.0/24']\n        # SELinux module\n        edpm_selinux_mode: enforcing\n        plan: overcloud\nEOF\n</code></pre> <ul> <li>Deploy OpenStackDataPlaneDeployment:</li> </ul> <pre><code>oc apply -f - &lt;&lt;EOF\napiVersion: dataplane.openstack.org/v1beta1\nkind: OpenStackDataPlaneDeployment\nmetadata:\n  name: openstack\nspec:\n  nodeSets:\n  - openstack\nEOF\n</code></pre>"},{"location":"openstack/edpm_adoption/#post-checks","title":"Post-checks","text":"<ul> <li> <p>Check if all the Ansible EE pods reaches <code>Completed</code> status:</p> <p><pre><code># watching the pods\nwatch oc get pod -l app=openstackansibleee\n</code></pre> <pre><code># following the ansible logs with:\noc logs -l app=openstackansibleee -f --max-log-requests 10\n</code></pre></p> </li> <li> <p>Wait for the dataplane node set to reach the Ready status:     <pre><code>oc wait --for condition=Ready osdpns/openstack --timeout=30m\n</code></pre></p> </li> </ul>"},{"location":"openstack/glance_adoption/","title":"Glance adoption","text":"<p>Adopting Glance means that an existing <code>OpenStackControlPlane</code> CR, where Glance is supposed to be disabled, should be patched to start the service with the configuration parameters provided by the source environment.</p> <p>When the procedure is over, the expectation is to see the <code>GlanceAPI</code> service up and running: the <code>Keystone endpoints</code> should be updated and the same backend of the source Cloud will be available. If the conditions above are met, the adoption is considered concluded.</p> <p>This guide also assumes that:</p> <ol> <li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li> <li>A <code>SNO</code> / <code>CodeReadyContainers</code> is running on the other side;</li> <li>(optional) an internal/external <code>Ceph</code> cluster is reachable by both <code>crc</code> and <code>TripleO</code></li> </ol>"},{"location":"openstack/glance_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, MariaDB and Keystone   should be already adopted.</li> </ul>"},{"location":"openstack/glance_adoption/#procedure-glance-adoption","title":"Procedure - Glance adoption","text":"<p>As already done for Keystone, the Glance Adoption follows the same pattern.</p>"},{"location":"openstack/glance_adoption/#using-local-storage-backend","title":"Using local storage backend","text":"<p>When Glance should be deployed with local storage backend (not Ceph), patch OpenStackControlPlane to deploy Glance:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  glance:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      storageClass: \"local-storage\"\n      storageRequest: 10G\n      glanceAPIInternal:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n        networkAttachments:\n        - storage\n      glanceAPIExternal:\n        networkAttachments:\n        - storage\n'\n</code></pre>"},{"location":"openstack/glance_adoption/#using-ceph-storage-backend","title":"Using Ceph storage backend","text":"<p>If a Ceph backend is used, the <code>customServiceConfig</code> parameter should be used to inject the right configuration to the <code>GlanceAPI</code> instance.</p> <p>Make sure the Ceph-related secret (<code>ceph-conf-files</code>) was created in the <code>openstack</code> namespace and that the <code>extraMounts</code> property of the <code>OpenStackControlPlane</code> CR has been configured properly. These tasks are described in an earlier Adoption step Ceph storage backend configuration.</p> <pre><code>cat &lt;&lt; EOF &gt; glance_patch.yaml\nspec:\n  glance:\n    enabled: true\n    template:\n      databaseInstance: openstack\n      customServiceConfig: |\n        [DEFAULT]\n        enabled_backends=default_backend:rbd\n        [glance_store]\n        default_backend=default_backend\n        [default_backend]\n        rbd_store_ceph_conf=/etc/ceph/ceph.conf\n        rbd_store_user=openstack\n        rbd_store_pool=images\n        store_description=Ceph glance store backend.\n      storageClass: \"local-storage\"\n      storageRequest: 10G\n      glanceAPIInternal:\n        override:\n          service:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n        networkAttachments:\n        - storage\n      glanceAPIExternal:\n        networkAttachments:\n        - storage\nEOF\n</code></pre> <p>If you have previously backup your Openstack services configuration file from the old environment: pull openstack configuration os-diff you can use os-diff to compare and make sure the configuration is correct.</p> <pre><code>pushd os-diff\n./os-diff cdiff --service glance -c /tmp/collect_tripleo_configs/glance/etc/glance/glance-api.conf -o glance_patch.yaml\n</code></pre> <p>This will producre the difference between both ini configuration files.</p> <p>Patch OpenStackControlPlane to deploy Glance with Ceph backend:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch-file glance_patch.yaml\n</code></pre>"},{"location":"openstack/glance_adoption/#post-checks","title":"Post-checks","text":""},{"location":"openstack/glance_adoption/#test-the-glance-service-from-the-openstack-cli","title":"Test the glance service from the OpenStack CLI","text":"<p>You can compare and make sure the configuration has been correctly applied to the glance pods by running</p> <pre><code>./os-diff cdiff --service glance -c /etc/glance/glance.conf.d/02-config.conf  -o glance_patch.yaml --frompod -p glance-api\n</code></pre> <p>If no line appear, then the configuration is correctly done.</p> <p>Inspect the resulting glance pods:</p> <pre><code>GLANCE_POD=`oc get pod |grep glance-external-api | cut -f 1 -d' '`\noc exec -t $GLANCE_POD -c glance-api -- cat /etc/glance/glance.conf.d/02-config.conf\n\n[DEFAULT]\nenabled_backends=default_backend:rbd\n[glance_store]\ndefault_backend=default_backend\n[default_backend]\nrbd_store_ceph_conf=/etc/ceph/ceph.conf\nrbd_store_user=openstack\nrbd_store_pool=images\nstore_description=Ceph glance store backend.\n\noc exec -t $GLANCE_POD -c glance-api -- ls /etc/ceph\nceph.client.openstack.keyring\nceph.conf\n</code></pre> <p>Ceph secrets are properly mounted, at this point let's move to the OpenStack CLI and check the service is active and the endpoints are properly updated.</p> <pre><code>(openstack)$ service list | grep image\n\n| fc52dbffef36434d906eeb99adfc6186 | glance    | image        |\n\n(openstack)$ endpoint list | grep image\n\n| 569ed81064f84d4a91e0d2d807e4c1f1 | regionOne | glance       | image        | True    | internal  | http://glance-internal-openstack.apps-crc.testing   |\n| 5843fae70cba4e73b29d4aff3e8b616c | regionOne | glance       | image        | True    | public    | http://glance-public-openstack.apps-crc.testing     |\n| 709859219bc24ab9ac548eab74ad4dd5 | regionOne | glance       | image        | True    | admin     | http://glance-admin-openstack.apps-crc.testing      |\n</code></pre> <p>Check the images that we previously listed in the source Cloud are available in the adopted service:</p> <pre><code>(openstack)$ image list\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| c3158cad-d50b-452f-bec1-f250562f5c1f | cirros | active |\n+--------------------------------------+--------+--------+\n</code></pre>"},{"location":"openstack/glance_adoption/#image-upload","title":"Image upload","text":"<p>We can test that an image can be created on from the adopted service.</p> <pre><code>(openstack)$ alias openstack=\"oc exec -t openstackclient -- openstack\"\n(openstack)$ curl -L -o /tmp/cirros-0.5.2-x86_64-disk.img http://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img\n    qemu-img convert -O raw /tmp/cirros-0.5.2-x86_64-disk.img /tmp/cirros-0.5.2-x86_64-disk.img.raw\n    openstack image create --container-format bare --disk-format raw --file /tmp/cirros-0.5.2-x86_64-disk.img.raw cirros2\n    openstack image list\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n100   273  100   273    0     0   1525      0 --:--:-- --:--:-- --:--:--  1533\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n100 15.5M  100 15.5M    0     0  17.4M      0 --:--:-- --:--:-- --:--:-- 17.4M\n\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n| Field            | Value                                                                                                                                      |\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n| container_format | bare                                                                                                                                       |\n| created_at       | 2023-01-31T21:12:56Z                                                                                                                       |\n| disk_format      | raw                                                                                                                                        |\n| file             | /v2/images/46a3eac1-7224-40bc-9083-f2f0cd122ba4/file                                                                                       |\n| id               | 46a3eac1-7224-40bc-9083-f2f0cd122ba4                                                                                                       |\n| min_disk         | 0                                                                                                                                          |\n| min_ram          | 0                                                                                                                                          |\n| name             | cirros                                                                                                                                     |\n| owner            | 9f7e8fdc50f34b658cfaee9c48e5e12d                                                                                                           |\n| properties       | os_hidden='False', owner_specified.openstack.md5='', owner_specified.openstack.object='images/cirros', owner_specified.openstack.sha256='' |\n| protected        | False                                                                                                                                      |\n| schema           | /v2/schemas/image                                                                                                                          |\n| status           | queued                                                                                                                                     |\n| tags             |                                                                                                                                            |\n| updated_at       | 2023-01-31T21:12:56Z                                                                                                                       |\n| visibility       | shared                                                                                                                                     |\n+------------------+--------------------------------------------------------------------------------------------------------------------------------------------+\n\n+--------------------------------------+--------+--------+\n| ID                                   | Name   | Status |\n+--------------------------------------+--------+--------+\n| 46a3eac1-7224-40bc-9083-f2f0cd122ba4 | cirros2| active |\n| c3158cad-d50b-452f-bec1-f250562f5c1f | cirros | active |\n+--------------------------------------+--------+--------+\n\n\n(openstack)$ oc rsh ceph\nsh-4.4$ ceph -s\nr  cluster:\n    id:     432d9a34-9cee-4109-b705-0c59e8973983\n    health: HEALTH_OK\n\n  services:\n    mon: 1 daemons, quorum a (age 4h)\n    mgr: a(active, since 4h)\n    osd: 1 osds: 1 up (since 4h), 1 in (since 4h)\n\n  data:\n    pools:   5 pools, 160 pgs\n    objects: 46 objects, 224 MiB\n    usage:   247 MiB used, 6.8 GiB / 7.0 GiB avail\n    pgs:     160 active+clean\n\nsh-4.4$ rbd -p images ls\n46a3eac1-7224-40bc-9083-f2f0cd122ba4\nc3158cad-d50b-452f-bec1-f250562f5c1f\n</code></pre>"},{"location":"openstack/heat_adoption/","title":"Heat adoption","text":"<p>Adopting Heat means that an existing <code>OpenStackControlPlane</code> CR, where Heat is supposed to be disabled, should be patched to start the service with the configuration parameters provided by the source environment.</p> <p>After the adoption process has been completed, a user can expect that they will then have CR's for <code>Heat</code>, <code>HeatAPI</code>, <code>HeatEngine</code> and <code>HeatCFNAPI</code>. Additionally, a user should have endpoints created within Keystone to facilitate the above mentioned servies.</p> <p>This guide also assumes that:</p> <ol> <li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li> <li>A OpenShift environment is running on the other side.</li> </ol>"},{"location":"openstack/heat_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, MariaDB and Keystone   should be already adopted.</li> <li>In addition, if your existing Heat stacks contain resources from other services   such as Neutron, Nova, Swift, etc. Those services should be adopted first before   trying to adopt Heat.</li> </ul>"},{"location":"openstack/heat_adoption/#procedure-heat-adoption","title":"Procedure - Heat adoption","text":"<p>As already done for Keystone, the Heat Adoption follows a similar pattern.</p> <p>Patch the <code>osp-secret</code> to update the <code>HeatAuthEncryptionKey</code> and <code>HeatPassword</code>. This needs to match what you have configured in the existing TripleO Heat configuration.</p> <p>You can retrieve and verify the existing <code>auth_encryption_key</code> and <code>service</code> passwords via: <pre><code>[stack@rhosp17 ~]$ grep -E 'HeatPassword|HeatAuth' ~/overcloud-deploy/overcloud/overcloud-passwords.yaml\n  HeatAuthEncryptionKey: Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2\n  HeatPassword: dU2N0Vr2bdelYH7eQonAwPfI3\n</code></pre></p> <p>And verifying on one of the Controllers that this is indeed the value in use: <pre><code>[stack@rhosp17 ~]$ ansible -i overcloud-deploy/overcloud/config-download/overcloud/tripleo-ansible-inventory.yaml overcloud-controller-0 -m shell -a \"grep auth_encryption_key /var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf | grep -Ev '^#|^$'\" -b\novercloud-controller-0 | CHANGED | rc=0 &gt;&gt;\nauth_encryption_key=Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2\n</code></pre></p> <p>This password needs to be base64 encoded and added to the <code>osp-secret</code> <pre><code>\u276f echo Q60Hj8PqbrDNu2dDCbyIQE2dibpQUPg2 | base64\nUTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK\n\n\u276f oc patch secret osp-secret --type='json' -p='[{\"op\" : \"replace\" ,\"path\" : \"/data/HeatAuthEncryptionKey\" ,\"value\" : \"UTYwSGo4UHFickROdTJkRENieUlRRTJkaWJwUVVQZzIK\"}]'\nsecret/osp-secret patched\n</code></pre></p> <p>Patch OpenStackControlPlane to deploy Heat:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  heat:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      secret: osp-secret\n      memcachedInstance: memcached\n      passwordSelectors:\n        authEncryptionKey: HeatAuthEncryptionKey\n        database: HeatDatabasePassword\n        service: HeatPassword\n'\n</code></pre>"},{"location":"openstack/heat_adoption/#post-checks","title":"Post-checks","text":"<p>Ensure all of the CR's reach the \"Setup Complete\" state: <pre><code>\u276f oc get Heat,HeatAPI,HeatEngine,HeatCFNAPI\nNAME                           STATUS   MESSAGE\nheat.heat.openstack.org/heat   True     Setup complete\n\nNAME                                  STATUS   MESSAGE\nheatapi.heat.openstack.org/heat-api   True     Setup complete\n\nNAME                                        STATUS   MESSAGE\nheatengine.heat.openstack.org/heat-engine   True     Setup complete\n\nNAME                                        STATUS   MESSAGE\nheatcfnapi.heat.openstack.org/heat-cfnapi   True     Setup complete\n</code></pre></p>"},{"location":"openstack/heat_adoption/#check-that-heat-service-is-registered-in-keystone","title":"Check that Heat service is registered in Keystone","text":"<pre><code> oc exec -it openstackclient -- openstack service list -c Name -c Type\n+------------+----------------+\n| Name       | Type           |\n+------------+----------------+\n| heat       | orchestration  |\n| glance     | image          |\n| heat-cfn   | cloudformation |\n| ceilometer | Ceilometer     |\n| keystone   | identity       |\n| placement  | placement      |\n| cinderv3   | volumev3       |\n| nova       | compute        |\n| neutron    | network        |\n+------------+----------------+\n</code></pre> <pre><code>\u276f oc exec -it openstackclient -- openstack endpoint list --service=heat -f yaml\n- Enabled: true\n  ID: 1da7df5b25b94d1cae85e3ad736b25a5\n  Interface: public\n  Region: regionOne\n  Service Name: heat\n  Service Type: orchestration\n  URL: http://heat-api-public-openstack-operators.apps.okd.bne-shift.net/v1/%(tenant_id)s\n- Enabled: true\n  ID: 414dd03d8e9d462988113ea0e3a330b0\n  Interface: internal\n  Region: regionOne\n  Service Name: heat\n  Service Type: orchestration\n  URL: http://heat-api-internal.openstack-operators.svc:8004/v1/%(tenant_id)s\n</code></pre>"},{"location":"openstack/heat_adoption/#check-heat-engine-services-are-up","title":"Check Heat engine services are up","text":"<pre><code> oc exec -it openstackclient -- openstack orchestration service list -f yaml\n- Binary: heat-engine\n  Engine ID: b16ad899-815a-4b0c-9f2e-e6d9c74aa200\n  Host: heat-engine-6d47856868-p7pzz\n  Hostname: heat-engine-6d47856868-p7pzz\n  Status: up\n  Topic: engine\n  Updated At: '2023-10-11T21:48:01.000000'\n- Binary: heat-engine\n  Engine ID: 887ed392-0799-4310-b95c-ac2d3e6f965f\n  Host: heat-engine-6d47856868-p7pzz\n  Hostname: heat-engine-6d47856868-p7pzz\n  Status: up\n  Topic: engine\n  Updated At: '2023-10-11T21:48:00.000000'\n- Binary: heat-engine\n  Engine ID: 26ed9668-b3f2-48aa-92e8-2862252485ea\n  Host: heat-engine-6d47856868-p7pzz\n  Hostname: heat-engine-6d47856868-p7pzz\n  Status: up\n  Topic: engine\n  Updated At: '2023-10-11T21:48:00.000000'\n- Binary: heat-engine\n  Engine ID: 1011943b-9fea-4f53-b543-d841297245fd\n  Host: heat-engine-6d47856868-p7pzz\n  Hostname: heat-engine-6d47856868-p7pzz\n  Status: up\n  Topic: engine\n  Updated At: '2023-10-11T21:48:01.000000'\n</code></pre>"},{"location":"openstack/heat_adoption/#verify-you-can-now-see-your-heat-stacks-again","title":"Verify you can now see your Heat stacks again","text":"<p>We can now test that user can create networks, subnets, ports, routers etc.</p> <pre><code>\u276f openstack stack list -f yaml\n- Creation Time: '2023-10-11T22:03:20Z'\n  ID: 20f95925-7443-49cb-9561-a1ab736749ba\n  Project: 4eacd0d1cab04427bc315805c28e66c9\n  Stack Name: test-networks\n  Stack Status: CREATE_COMPLETE\n  Updated Time: null\n</code></pre>"},{"location":"openstack/horizon_adoption/","title":"Horizon adoption","text":""},{"location":"openstack/horizon_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, Memcached and   keystone should be already adopted.</li> </ul>"},{"location":"openstack/horizon_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/horizon_adoption/#procedure-horizon-adoption","title":"Procedure - Horizon adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Horizon:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  horizon:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      memcachedInstance: memcached\n      secret: osp-secret\n'\n</code></pre>"},{"location":"openstack/horizon_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Horizon instance is successfully deployed and ready</li> </ul> <pre><code>oc get horizon\n</code></pre> <ul> <li>Check that dashboard is reachable and returns status code <code>200</code></li> </ul> <pre><code>PUBLIC_URL=$(oc get horizon horizon -o jsonpath='{.status.endpoint}')\ncurl --silent --output /dev/stderr --head --write-out \"%{http_code}\" \"$PUBLIC_URL/dashboard/auth/login/?next=/dashboard/\" | grep 200\n</code></pre>"},{"location":"openstack/ironic_adoption/","title":"Ironic adoption","text":""},{"location":"openstack/ironic_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/ironic_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/ironic_adoption/#pre-checks","title":"Pre-checks","text":"<p>TODO</p>"},{"location":"openstack/ironic_adoption/#procedure-ironic-adoption","title":"Procedure - Ironic adoption","text":"<p>TODO</p>"},{"location":"openstack/ironic_adoption/#post-checks","title":"Post-checks","text":"<p>TODO</p>"},{"location":"openstack/keystone_adoption/","title":"Keystone adoption","text":""},{"location":"openstack/keystone_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/keystone_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/keystone_adoption/#pre-checks","title":"Pre-checks","text":""},{"location":"openstack/keystone_adoption/#procedure-keystone-adoption","title":"Procedure - Keystone adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Keystone:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  keystone:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n      databaseInstance: openstack\n      secret: osp-secret\n'\n</code></pre> <ul> <li>Create alias to use <code>openstack</code> command in the adopted deployment:</li> </ul> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n</code></pre> <ul> <li>Clean up old services and endpoints that still point to the old   control plane (everything except Keystone service and endpoints):</li> </ul> <pre><code>openstack endpoint list | grep keystone | awk '/admin/{ print $2; }' | xargs ${BASH_ALIASES[openstack]} endpoint delete || true\n\nfor service in cinderv3 glance neutron nova placement swift; do\n  openstack service list | awk \"/ $service /{ print \\$2; }\" | xargs ${BASH_ALIASES[openstack]} service delete || true\ndone\n</code></pre>"},{"location":"openstack/keystone_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Keystone endpoints are defined and pointing to the podified   FQDNs:</li> </ul> <pre><code>openstack endpoint list | grep keystone\n</code></pre>"},{"location":"openstack/mariadb_copy/","title":"MariaDB data copy","text":"<p>This document describes how to move the databases from the original OpenStack deployment to the MariaDB instances in the OpenShift cluster.</p>"},{"location":"openstack/mariadb_copy/#prerequisites","title":"Prerequisites","text":"<ul> <li> <p>Make sure the previous Adoption steps have been performed successfully.</p> </li> <li> <p>The OpenStackControlPlane resource must be already created at this point.</p> </li> <li> <p>Podified MariaDB and RabbitMQ are running. No other podified     control plane services are running.</p> </li> <li> <p>OpenStack services have been stopped</p> </li> <li> <p>There must be network routability between:</p> <ul> <li> <p>The adoption host and the original MariaDB.</p> </li> <li> <p>The adoption host and the podified MariaDB.</p> </li> <li> <p>Note that this routability requirement may change in the   future, e.g. we may require routability from the original MariaDB to   podified MariaDB.</p> </li> </ul> </li> <li> <p>Podman package is installed</p> </li> <li> <p><code>CONTROLLER1_SSH</code>, <code>CONTROLLER2_SSH</code>, and <code>CONTROLLER3_SSH</code> are configured.</p> </li> </ul>"},{"location":"openstack/mariadb_copy/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified\n\nPODIFIED_MARIADB_IP=$(oc get svc --selector \"cr=mariadb-openstack\" -ojsonpath='{.items[0].spec.clusterIP}')\nPODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector \"cr=mariadb-openstack-cell1\" -ojsonpath='{.items[0].spec.clusterIP}')\nPODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)\n\n# Replace with your environment's MariaDB IP:\nSOURCE_MARIADB_IP=192.168.122.100\nSOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }')\n\n# The CHARACTER_SET and collation should match the source DB\n# if the do not then it will break foreign key relationships\n# for any tables that are created in the future as part of db sync\nCHARACTER_SET=utf8\nCOLLATION=utf8_general_ci\n</code></pre>"},{"location":"openstack/mariadb_copy/#pre-checks","title":"Pre-checks","text":"<ul> <li>Test connection to the original DB (show databases):</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \\\n    mysql -h \"$SOURCE_MARIADB_IP\" -uroot \"-p$SOURCE_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\n</code></pre> <ul> <li>Run mysqlcheck on the original DB to look for things that are not OK:</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \\\n    mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root \"-p$SOURCE_DB_ROOT_PASSWORD\" | grep -v OK\n</code></pre> <ul> <li>Test connection to podified DBs (show databases):</li> </ul> <pre><code>oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\n    mysql -h \"$PODIFIED_MARIADB_IP\" -uroot \"-p$PODIFIED_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\n    mysql -h \"$PODIFIED_CELL1_MARIADB_IP\" -uroot \"-p$PODIFIED_DB_ROOT_PASSWORD\" -e 'SHOW databases;'\n</code></pre>"},{"location":"openstack/mariadb_copy/#procedure-data-copy","title":"Procedure - data copy","text":"<ul> <li>Create a temporary folder to store DB dumps and make sure it's the   working directory for the following steps:</li> </ul> <pre><code>mkdir ~/adoption-db\ncd ~/adoption-db\n</code></pre> <ul> <li>Create a dump of the original databases:</li> </ul> <pre><code>podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash &lt;&lt;EOF\n\n# Note we do not want to dump the information and performance schema tables so we filter them\nmysql -h ${SOURCE_MARIADB_IP} -u root \"-p${SOURCE_DB_ROOT_PASSWORD}\" -N -e 'show databases' | grep -E -v 'schema|mysql' | while read dbname; do\n    echo \"Dumping \\${dbname}\"\n    mysqldump -h $SOURCE_MARIADB_IP -uroot \"-p$SOURCE_DB_ROOT_PASSWORD\" \\\n        --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \\\n        \"\\${dbname}\" &gt; \"\\${dbname}\".sql\ndone\n\nEOF\n</code></pre> <ul> <li>Restore the databases from .sql files into the podified MariaDB:</li> </ul> <pre><code># db schemas to rename on import\ndeclare -A db_name_map\ndb_name_map[\"nova\"]=\"nova_cell1\"\ndb_name_map[\"ovs_neutron\"]=\"neutron\"\n\n# db servers to import into\ndeclare -A db_server_map\ndb_server_map[\"default\"]=${PODIFIED_MARIADB_IP}\ndb_server_map[\"nova_cell1\"]=${PODIFIED_CELL1_MARIADB_IP}\n\n# db server root password map\ndeclare -A db_server_password_map\ndb_server_password_map[\"default\"]=${PODIFIED_DB_ROOT_PASSWORD}\ndb_server_password_map[\"nova_cell1\"]=${PODIFIED_DB_ROOT_PASSWORD}\n\nall_db_files=$(ls *.sql)\nfor db_file in ${all_db_files}; do\n    db_name=$(echo ${db_file} | awk -F'.' '{ print $1; }')\n    if [[ -v \"db_name_map[${db_name}]\" ]]; then\n        echo \"renaming ${db_name} to ${db_name_map[${db_name}]}\"\n        db_name=${db_name_map[${db_name}]}\n    fi\n    db_server=${db_server_map[\"default\"]}\n    if [[ -v \"db_server_map[${db_name}]\" ]]; then\n        db_server=${db_server_map[${db_name}]}\n    fi\n    db_password=${db_server_password_map[\"default\"]}\n    if [[ -v \"db_server_password_map[${db_name}]\" ]]; then\n        db_password=${db_server_password_map[${db_name}]}\n    fi\n    echo \"creating ${db_name} in ${db_server}\"\n    container_name=$(echo \"mariadb-client-${db_name}-create\" | sed 's/_/-/g')\n    oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \\\n        mysql -h \"${db_server}\" -uroot \"-p${db_password}\" &lt;&lt; EOF\nCREATE DATABASE IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};\nEOF\n    echo \"importing ${db_name} into ${db_server}\"\n    container_name=$(echo \"mariadb-client-${db_name}-restore\" | sed 's/_/-/g')\n    oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \\\n        mysql -h \"${db_server}\" -uroot \"-p${db_password}\" \"${db_name}\" &lt; \"${db_file}\"\ndone\n</code></pre>"},{"location":"openstack/mariadb_copy/#post-checks","title":"Post-checks","text":"<ul> <li>Check that the databases were imported correctly:</li> </ul> <pre><code>oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep keystone\n# ensure neutron db is renamed from ovs_neutron\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep neutron\n# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1\noc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \\\nmysql -h \"${PODIFIED_CELL1_MARIADB_IP}\" -uroot \"-p${PODIFIED_DB_ROOT_PASSWORD}\" -e 'SHOW databases;' \\\n    | grep nova_cell1\n</code></pre> <ul> <li>During the pre/post checks the pod <code>mariadb-client</code> might have returned a pod security warning   related to the <code>restricted:latest</code> security context constraint. This is due to default security   context constraints and will not prevent pod creation by the admission controller. You'll see a   warning for the short-lived pod but it will not interfere with functionality.   For more info visit here</li> </ul>"},{"location":"openstack/neutron_adoption/","title":"Neutron adoption","text":"<p>Adopting Neutron means that an existing <code>OpenStackControlPlane</code> CR, where Neutron is supposed to be disabled, should be patched to start the service with the configuration parameters provided by the source environment.</p> <p>When the procedure is over, the expectation is to see the <code>NeutronAPI</code> service up and running: the <code>Keystone endpoints</code> should be updated and the same backend of the source Cloud will be available. If the conditions above are met, the adoption is considered concluded.</p> <p>This guide also assumes that:</p> <ol> <li>A <code>TripleO</code> environment (the source Cloud) is running on one side;</li> <li>A <code>SNO</code> / <code>CodeReadyContainers</code> is running on the other side.</li> </ol>"},{"location":"openstack/neutron_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, MariaDB and Keystone   should be already adopted.</li> </ul>"},{"location":"openstack/neutron_adoption/#procedure-neutron-adoption","title":"Procedure - Neutron adoption","text":"<p>As already done for Keystone, the Neutron Adoption follows the same pattern.</p> <p>Patch OpenStackControlPlane to deploy Neutron:</p> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  neutron:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n      databaseInstance: openstack\n      secret: osp-secret\n      networkAttachments:\n      - internalapi\n'\n</code></pre>"},{"location":"openstack/neutron_adoption/#post-checks","title":"Post-checks","text":""},{"location":"openstack/neutron_adoption/#inspect-the-resulting-neutron-pods","title":"Inspect the resulting neutron pods","text":"<pre><code>NEUTRON_API_POD=`oc get pods -l service=neutron | tail -n 1 | cut -f 1 -d' '`\noc exec -t $NEUTRON_API_POD -c neutron-api -- cat /etc/neutron/neutron.conf\n</code></pre>"},{"location":"openstack/neutron_adoption/#check-that-neutron-api-service-is-registered-in-keystone","title":"Check that Neutron API service is registered in Keystone","text":"<pre><code>openstack service list | grep network\n</code></pre> <pre><code>openstack endpoint list | grep network\n\n| 6a805bd6c9f54658ad2f24e5a0ae0ab6 | regionOne | neutron      | network      | True    | public    | http://neutron-public-openstack.apps-crc.testing  |\n| b943243e596847a9a317c8ce1800fa98 | regionOne | neutron      | network      | True    | internal  | http://neutron-internal.openstack.svc:9696        |\n| f97f2b8f7559476bb7a5eafe3d33cee7 | regionOne | neutron      | network      | True    | admin     | http://192.168.122.99:9696                        |\n</code></pre>"},{"location":"openstack/neutron_adoption/#create-sample-resources","title":"Create sample resources","text":"<p>We can now test that user can create networks, subnets, ports, routers etc.</p> <pre><code>openstack network create net\nopenstack subnet create --network net --subnet-range 10.0.0.0/24 subnet\nopenstack router create router\n</code></pre> <p>NOTE: this page should be expanded to include information on SR-IOV adoption.</p>"},{"location":"openstack/node-selector/","title":"Node Selector","text":"<p>There are a variety of reasons why we may want to restrict the nodes where OpenStack services can be placed:</p> <ul> <li>Hardware requirements: System memory, Disk space, Cores, HBAs</li> <li>Limit the impact of the OpenStack services on other OpenShift workloads.</li> <li>Avoid collocating OpenStack services.</li> </ul> <p>The mechanism provided by the OpenStack operators to achieve this is through the use of labels.</p> <p>We would either label the OpenShift nodes or use existing labels they already have, and then use those labels in the OpenStack manifests in the <code>nodeSelector</code> field.</p> <p>The <code>nodeSelector</code> field in the OpenStack manifests follows the standard OpenShift <code>nodeSelector</code> field, please refer to the OpenShift documentation on the matter additional information.</p> <p>This field is present at all the different levels of the OpenStack manifests:</p> <ul> <li>Deployment: The <code>OpenStackControlPlane</code> object.</li> <li>Component: For example the <code>cinder</code> element in the <code>OpenStackControlPlane</code>.</li> <li>Service: For example the <code>cinderVolume</code> element within the <code>cinder</code> element   in the <code>OpenStackControlPlane</code>.</li> </ul> <p>This allows a fine grained control of the placement of the OpenStack services with minimal repetition.</p> <p>Values of the <code>nodeSelector</code> are propagated to the next levels unless they are overwritten. This means that a <code>nodeSelector</code> value at the deployment level will affect all the OpenStack services.</p> <p>For example we can add label <code>type: openstack</code> to any 3 OpenShift nodes:</p> <pre><code>$ oc label nodes worker0 type=openstack\n$ oc label nodes worker1 type=openstack\n$ oc label nodes worker2 type=openstack\n</code></pre> <p>And then in our <code>OpenStackControlPlane</code> we can use the label to place all the services in those 3 nodes:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  nodeSelector:\n    type: openstack\n&lt; . . . &gt;\n</code></pre> <p>What if we don't mind where any OpenStack services go but the cinder volume and backup services because we are using FC and we only have HBAs on a subset of nodes? Then we can just use the selector on for those specific services, which for the sake of this example we'll assume they have the label <code>fc_card: true</code>:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  secret: osp-secret\n  storageClass: local-storage\n  cinder:\n    template:\n      cinderVolumes:\n          pure_fc:\n            nodeSelector:\n              fc_card: true\n&lt; . . . &gt;\n          lvm-iscsi:\n            nodeSelector:\n              fc_card: true\n&lt; . . . &gt;\n      cinderBackup:\n          nodeSelector:\n            fc_card: true\n&lt; . . . &gt;\n</code></pre> <p>The Cinder operator does not currently have the possibility of defining the <code>nodeSelector</code> in <code>cinderVolumes</code>, so we need to specify it on each of the backends.</p> <p>It's possible to leverage labels added by the node feature discovery operator to place OpenStack services.</p>"},{"location":"openstack/node-selector/#machineconfig","title":"MachineConfig","text":"<p>Some services require us to have services or kernel modules running on the hosts where they run, for example <code>iscsid</code> or <code>multipathd</code> daemons, or the <code>nvme-fabrics</code> kernel module.</p> <p>For those cases we'll use <code>MachineConfig</code> manifests, and if we are restricting the nodes we are placing the OpenStack services using the <code>nodeSelector</code> then we'll also want to limit where the <code>MachineConfig</code> is applied.</p> <p>To define where the <code>MachineConfig</code> can be applied we'll need to use a <code>MachineConfigPool</code> that links the <code>MachineConfig</code> to the nodes.</p> <p>For example to be able to limit <code>MachineConfig</code> to the 3 OpenShift nodes we marked with the <code>type: openstack</code> label we would create the <code>MachineConfigPool</code> like this:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfigPool\nmetadata:\n  name: openstack\nspec:\n  machineConfigSelector:\n    matchLabels:\n      machineconfiguration.openshift.io/role: openstack\n  nodeSelector:\n    matchLabels:\n      type: openstack\n</code></pre> <p>And then we could use it in the <code>MachineConfig</code>:</p> <pre><code>apiVersion: machineconfiguration.openshift.io/v1\nkind: MachineConfig\nmetadata:\n  labels:\n    machineconfiguration.openshift.io/role: openstack\n&lt; . . . &gt;\n</code></pre> <p>Refer to the OpenShift documentation for additional information on <code>MachineConfig</code> and <code>MachineConfigPools</code></p> <p>WARNING: Applying a <code>MachineConfig</code> to an OpenShift node will make the node reboot.</p>"},{"location":"openstack/ovn_adoption/","title":"OVN data migration","text":"<p>This document describes how to move OVN northbound and southbound databases from the original OpenStack deployment to ovsdb-server instances running in the OpenShift cluster.</p>"},{"location":"openstack/ovn_adoption/#rationale","title":"Rationale","text":"<p>While it may be argued that the podified Neutron ML2/OVN driver and OVN northd service will reconstruct the databases on startup, the reconstruction may be time consuming on large existing clusters. The procedure below allows to speed up data migration and avoid unnecessary data plane disruptions due to incomplete OpenFlow table contents.</p>"},{"location":"openstack/ovn_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Make sure the previous Adoption steps have been performed successfully.</li> <li>The OpenStackControlPlane resource must be already created at this point.</li> <li>NetworkAttachmentDefinition CRDs for the original cluster are already     defined. Specifically, openstack/internalapi network is defined.</li> <li>Podified MariaDB and RabbitMQ may already run. Neutron and OVN are not     running yet.</li> <li>Original OVN is older or equal to the podified version.</li> <li>There must be network routability between:<ul> <li>The adoption host and the original OVN.</li> <li>The adoption host and the podified OVN.</li> </ul> </li> </ul>"},{"location":"openstack/ovn_adoption/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment:</p> <pre><code>OVSDB_IMAGE=quay.io/podified-antelope-centos9/openstack-ovn-base:current-podified\nSOURCE_OVSDB_IP=172.17.1.49\n\n# ssh commands to reach the original controller machines\nCONTROLLER_SSH=\"ssh -F ~/director_standalone/vagrant_ssh_config vagrant@standalone\"\n\n# ssh commands to reach the original compute machines\nCOMPUTE_SSH=\"ssh -F ~/director_standalone/vagrant_ssh_config vagrant@standalone\"\n</code></pre>"},{"location":"openstack/ovn_adoption/#procedure","title":"Procedure","text":"<ul> <li>Stop OVN northd on all original cluster controllers.</li> </ul> <pre><code>${CONTROLLER_SSH} sudo systemctl stop tripleo_ovn_cluster_northd.service\n</code></pre> <ul> <li>Backup OVN databases.</li> </ul> <pre><code>client=\"podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE ovsdb-client\"\n${client} backup tcp:$SOURCE_OVSDB_IP:6641 &gt; ovs-nb.db\n${client} backup tcp:$SOURCE_OVSDB_IP:6642 &gt; ovs-sb.db\n</code></pre> <ul> <li>Start podified OVN database services prior to import.</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  ovn:\n\u00a0 \u00a0 enabled: true\n    template:\n      ovnDBCluster:\n        ovndbcluster-nb:\n          dbType: NB\n          storageRequest: 10G\n          networkAttachment: internalapi\n        ovndbcluster-sb:\n          dbType: SB\n          storageRequest: 10G\n          networkAttachment: internalapi\n'\n</code></pre> <ul> <li>Fetch podified OVN IP addresses.</li> </ul> <pre><code>PODIFIED_OVSDB_NB_IP=$(kubectl get po ovsdbserver-nb-0 -o jsonpath='{.metadata.annotations.k8s\\.v1\\.cni\\.cncf\\.io/network-status}' | jq 'map(. | select(.name==\"openstack/internalapi\"))[0].ips[0]' | tr -d '\"')\nPODIFIED_OVSDB_SB_IP=$(kubectl get po ovsdbserver-sb-0 -o jsonpath='{.metadata.annotations.k8s\\.v1\\.cni\\.cncf\\.io/network-status}' | jq 'map(. | select(.name==\"openstack/internalapi\"))[0].ips[0]' | tr -d '\"')\n</code></pre> <ul> <li>Upgrade database schema for the backup files.</li> </ul> <pre><code>podman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client get-schema tcp:$PODIFIED_OVSDB_NB_IP:6641 &gt; ./ovs-nb.ovsschema &amp;&amp; ovsdb-tool convert ovs-nb.db ./ovs-nb.ovsschema\"\npodman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client get-schema tcp:$PODIFIED_OVSDB_SB_IP:6642 &gt; ./ovs-sb.ovsschema &amp;&amp; ovsdb-tool convert ovs-sb.db ./ovs-sb.ovsschema\"\n</code></pre> <ul> <li>Restore database backup to podified OVN database servers.</li> </ul> <pre><code>podman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client restore tcp:$PODIFIED_OVSDB_NB_IP:6641 &lt; ovs-nb.db\"\npodman run -it --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $OVSDB_IMAGE bash -c \"ovsdb-client restore tcp:$PODIFIED_OVSDB_SB_IP:6642 &lt; ovs-sb.db\"\n</code></pre> <ul> <li>Check that podified OVN databases contain objects from backup, e.g.:</li> </ul> <pre><code>oc exec -it ovsdbserver-nb-0 -- ovn-nbctl show\noc exec -it ovsdbserver-sb-0 -- ovn-sbctl list Chassis\n</code></pre> <ul> <li>Switch ovn-remote on compute nodes to point to the new podified database.</li> </ul> <pre><code>${COMPUTE_SSH} sudo podman exec -it ovn_controller ovs-vsctl set open . external_ids:ovn-remote=tcp:$PODIFIED_OVSDB_SB_IP:6642\n</code></pre> <p>You should now see the following warning in the <code>ovn_controller</code> container logs:</p> <pre><code>2023-03-16T21:40:35Z|03095|ovsdb_cs|WARN|tcp:172.17.1.50:6642: clustered database server has stale data; trying another server\n</code></pre> <ul> <li>Reset RAFT state for all compute ovn-controller instances.</li> </ul> <pre><code>${COMPUTE_SSH} sudo podman exec -it ovn_controller ovn-appctl -t ovn-controller sb-cluster-state-reset\n</code></pre> <p>This should complete connection of the controller process to the new remote. See in logs:</p> <pre><code>2023-03-16T21:42:31Z|03134|main|INFO|Resetting southbound database cluster state\n2023-03-16T21:42:33Z|03135|reconnect|INFO|tcp:172.17.1.50:6642: connected\n</code></pre> <ul> <li>Alternatively, just restart ovn-controller on original compute nodes.</li> </ul> <pre><code>$ ${COMPUTE_SSH} sudo systemctl restart tripleo_ovn_controller.service\n</code></pre> <ul> <li>Finally, you can start <code>ovn-northd</code> service that will keep OVN northbound and southbound databases in sync.</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  ovn:\n\u00a0 \u00a0 enabled: true\n    template:\n      ovnNorthd:\n        networkAttachment: internalapi\n'\n</code></pre>"},{"location":"openstack/placement_adoption/","title":"Placement adoption","text":""},{"location":"openstack/placement_adoption/#prerequisites","title":"Prerequisites","text":"<ul> <li>Previous Adoption steps completed. Notably, the service databases   must already be imported into the podified MariaDB.</li> </ul>"},{"location":"openstack/placement_adoption/#variables","title":"Variables","text":"<p>(There are no shell variables necessary currently.)</p>"},{"location":"openstack/placement_adoption/#procedure-placement-adoption","title":"Procedure - Placement adoption","text":"<ul> <li>Patch OpenStackControlPlane to deploy Placement:</li> </ul> <pre><code>oc patch openstackcontrolplane openstack --type=merge --patch '\nspec:\n  placement:\n    enabled: true\n    apiOverride:\n      route: {}\n    template:\n      databaseInstance: openstack\n      secret: osp-secret\n      override:\n        service:\n          internal:\n            metadata:\n              annotations:\n                metallb.universe.tf/address-pool: internalapi\n                metallb.universe.tf/allow-shared-ip: internalapi\n                metallb.universe.tf/loadBalancerIPs: 172.17.0.80\n            spec:\n              type: LoadBalancer\n'\n</code></pre>"},{"location":"openstack/placement_adoption/#post-checks","title":"Post-checks","text":"<ul> <li>See that Placement endpoints are defined and pointing to the   podified FQDNs and that Placement API responds.</li> </ul> <pre><code>alias openstack=\"oc exec -t openstackclient -- openstack\"\n\nopenstack endpoint list | grep placement\n\n\n# Without OpenStack CLI placement plugin installed:\nPLACEMENT_PUBLIC_URL=$(openstack endpoint list -c 'Service Name' -c 'Service Type' -c URL | grep placement | grep public | awk '{ print $6; }')\ncurl \"$PLACEMENT_PUBLIC_URL\"\n\n# With OpenStack CLI placement plugin installed:\nopenstack resource class list\n</code></pre>"},{"location":"openstack/planning/","title":"Planning the new deployment","text":"<p>Just like you did back when you installed your Director deployed OpenStack, the upgrade/migration to the podified OpenStack requires planning various aspects of the environment such as node roles, planning your network topology, and storage.</p> <p>In this document we cover some of this planning, but it is recommended to read the whole adoption guide before actually starting the process to be sure that there is a global understanding of the whole process.</p>"},{"location":"openstack/planning/#configurations","title":"Configurations","text":"<p>There is a fundamental difference between the Director and Operator deployments regarding the configuration of the services.</p> <p>In Director deployments many of the service configurations are abstracted by Director specific configuration options. A single Director option may trigger changes for multiple services and support for drivers (for example Cinder's) required patches to the Director code base.</p> <p>In Operator deployments this has changed to what we believe is a simpler approach: reduce the installer specific knowledge and leverage OpenShift and OpenStack service specific knowledge whenever possible.</p> <p>To this effect OpenStack services will have sensible defaults for OpenShift deployments and human operators will provide configuration snippets to provide necessary configuration, such as cinder backend configuration, or to override the defaults.</p> <p>This shortens the distance between a service specific configuration file (such as <code>cinder.conf</code>) and what the human operator provides in the manifests.</p> <p>These configuration snippets are passed to the operators in the different <code>customServiceConfig</code> sections available in the manifests, and then they are layered in the services available in the following levels. To illustrate this, if we were to set a configuration at the top Cinder level (<code>spec: cinder: template:</code>) then it would be applied to all the cinder services; for example to enable debug in all the cinder services we would do:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    template:\n      customServiceConfig: |\n        [DEFAULT]\n        debug = True\n&lt; . . . &gt;\n</code></pre> <p>If we only wanted to set it for one of the cinder services, for example the scheduler, then we would use the <code>cinderScheduler</code> section instead:</p> <pre><code>apiVersion: core.openstack.org/v1beta1\nkind: OpenStackControlPlane\nmetadata:\n  name: openstack\nspec:\n  cinder:\n    template:\n      cinderScheduler:\n        customServiceConfig: |\n          [DEFAULT]\n          debug = True\n&lt; . . . &gt;\n</code></pre> <p>In openshift it is not recommended to store sensitive information like the credentials to the cinder storage array in the CRs, so most OpenStack operators have a mechanism to use OpenShift's <code>Secrets</code> for sensitive configuration parameters of the services and then use then by reference in the <code>customServiceConfigSecrets</code> section which is analogous to the <code>customServiceConfig</code>.</p> <p>The contents of the <code>Secret</code> references passed in the <code>customServiceConfigSecrets</code> will have the same format as <code>customServiceConfig</code>: a snippet with the section/s and configuration options.</p> <p>When there are sensitive information in the service configuration then it becomes a matter of personal preference whether to store all the configuration in the <code>Secret</code> or only the sensitive parts, but remember that if we split the configuration between <code>Secret</code> and <code>customServiceConfig</code> we still need the section header (eg: <code>[DEFAULT]</code>) to be present in both places.</p> <p>Attention should be paid to each service's adoption process as they may have some particularities regarding their configuration.</p>"},{"location":"openstack/planning/#configuration-tooling","title":"Configuration tooling","text":"<p>In order to help users to handle the configuration for the TripleO and Openstack services the tool: https://github.com/openstack-k8s-operators/os-diff has been develop to compare the configuration files between the TripleO deployment and the next gen cloud. Make sure Golang is installed and configured on your env:</p> <pre><code>git clone https://github.com/openstack-k8s-operators/os-diff\npushd os-diff\nmake build\n</code></pre> <p>Then configure ansible.cfg and ssh-config file according to your environment:</p> <pre><code>Host *\n    IdentitiesOnly yes\n\nHost virthost\n    Hostname virthost\n    IdentityFile ~/.ssh/id_rsa\n    User root\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n\n\nHost standalone\n    Hostname standalone\n    IdentityFile ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa\n    User root\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n\nHost crc\n    Hostname crc\n    IdentityFile ~/.ssh/id_rsa\n    User stack\n    StrictHostKeyChecking no\n    UserKnownHostsFile=/dev/null\n</code></pre> <p>And test your connection:</p> <pre><code>ssh -F ssh.config standalone\n</code></pre>"},{"location":"openstack/planning/#node-roles","title":"Node Roles","text":"<p>In Director deployments we had 4 different standard roles for the nodes: <code>Controller</code>, <code>Compute</code>, <code>Ceph Storage</code>, <code>Swift Storage</code>, but in podified OpenStack we just make a distinction based on where things are running, in OpenShift or external to it.</p> <p>When adopting a Director OpenStack your <code>Compute</code> nodes will directly become external nodes, so there should not be much additional planning needed there.</p> <p>In many deployments being adopted the <code>Controller</code> nodes will require some thought because we'll have many OpenShift nodes where the controller services could run, and we have to decide which ones we want to use, how we are going to use them, and make sure those nodes are ready to run the services.</p> <p>In most deployments running OpenStack services on <code>master</code> nodes can have a seriously adverse impact on the OpenShift cluster, so we recommend placing OpenStack services on non <code>master</code> nodes.</p> <p>By default OpenStack Operators deploy OpenStack services on any worker node, but that is not necessarily what's best for all deployments, and there may be even services that won't even work deployed like that.</p> <p>When planing a deployment it's good to remember that not all the services on an OpenStack deployments are the same as they have very different requirements.</p> <p>Looking at the Cinder component we can clearly see different requirements for its services: the cinder-scheduler is a very light service with low memory, disk, network, and CPU usage; cinder-api service has a higher network usage due to resource listing requests; the cinder-volume service will have a high disk and network usage since many of its operations are in the data path (offline volume migration, create volume from image, etc.), and then we have the cinder-backup service which has high memory, network, and CPU (to compress data) requirements.</p> <p>We also have the Glance and Swift components that are in the data path, and let's not forget RabbitMQ and Galera services.</p> <p>Given these requirements it may be preferable not to let these services wander all over your OpenShift worker nodes with the possibility of impacting other workloads, or maybe you don't mind the light services wandering around but you want to pin down the heavy ones to a set of infrastructure nodes.</p> <p>There are also hardware restrictions to take into consideration, because if we are using a Fibre Channel (FC) Cinder backend we'll need the cinder-volume, cinder-backup, and maybe even the glance (if it's using Cinder as a backend) services to run on a OpenShift host that has an HBA.</p> <p>The OpenStack Operators allow a great deal of flexibility on where to run the OpenStack services, as we can use node labels to define which OpenShift nodes are eligible to run the different OpenStack services.  Refer to the Node Selector guide to learn more about using labels to define placement of the OpenStack services.</p> <p>TODO: Talk about Ceph Storage and Swift Storage nodes, HCI deployments, etc.</p>"},{"location":"openstack/planning/#network","title":"Network","text":"<p>TODO: Write about isolated networks, NetworkAttachmentDefinition, NetworkAttachmets, etc</p>"},{"location":"openstack/planning/#storage","title":"Storage","text":"<p>When looking into the storage in an OpenStack deployment we can differentiate 2 different kinds, the storage requirements of the services themselves and the storage used for the OpenStack users that thee services will manage.</p> <p>These requirements may drive our OpenShift node selection, as mentioned above, and may even require us to do some preparations on the OpenShift nodes before we can deploy the services.</p> <p>TODO: Galera, RabbitMQ, Swift, Glance, etc.</p>"},{"location":"openstack/planning/#cinder-requirements","title":"Cinder requirements","text":"<p>The Cinder service has both local storage used by the service and OpenStack user requirements.</p> <p>Local storage is used for example when downloading a glance image for the create volume from image operation, which can become considerable when having concurrent operations and not using cinder volume cache.</p> <p>In the Operator deployed OpenStack we now have an easy way to configure the location of the conversion directory to be an NFS share (using the extra volumes feature), something that needed to be done manually before.</p> <p>Even if it's an adoption and it may seem that there's nothing to consider regarding the Cinder backends, because we'll just be using the same ones we are using in our current deployment, we should still evaluate it, because it may not be so straightforward.</p> <p>First we need to check the transport protocol the Cinder backends are using: RBD, iSCSI, FC, NFS, NVMe-oF, etc.</p> <p>Once we know all the transport protocols we are using, we can proceed to make sure we are taking them into consideration when placing the Cinder services (as mentioned above in the Node Roles section) and the right storage transport related binaries are running on the OpenShift nodes.</p> <p>Detailed information about the specifics for each storage transport protocol can be found in the Cinder Adoption section.  Please take a good look at that document before proceeding to be able to plan the adoption better.</p>"},{"location":"openstack/pull_openstack_configuration/","title":"Pull Openstack configuration","text":"<p>Before starting to adoption workflow, we can start by pulling the configuration from the Openstack services and TripleO on our file system in order to backup the configuration files and then use it for later, during the configuration of the adopted services and for the record to compare and make sure nothing has been missed or misconfigured.</p> <p>Make sure you have pull the os-diff repository and configure according to your environment: Configure os-diff</p>"},{"location":"openstack/pull_openstack_configuration/#pull-configuration-from-a-tripleo-deployment","title":"Pull configuration from a TripleO deployment","text":"<p>Once you make sure the ssh connnection is confugred correctly and os-diff has been built, you can start to pull configuration from your Openstack services.</p> <p>All the services are describes in an Ansible role:</p> <p>collect_config vars</p> <p>Once you enabled the services you need (you can enable everything even if a services is not deployed) you can start to pull the Openstack services configuration files:</p> <pre><code>pushd os-diff\n./os-diff pull --cloud_engine=podman\n</code></pre> <p>The configuration will be pulled and stored in: <pre><code>/tmp/collect_tripleo_configs\n</code></pre></p> <p>And you provided another path with:</p> <pre><code>./os-diff pull --cloud_engine=podman -e local_working_dir=$HOME\n</code></pre> <p>Once the ansible playbook has been run, you should have into your local directory a directory per services</p> <pre><code>  \u25be tmp/\n    \u25be collect_tripleo_configs/\n      \u25be glance/\n</code></pre>"},{"location":"openstack/stop_openstack_services/","title":"Stop OpenStack services","text":"<p>Before we can start with the adoption we need to make sure that the OpenStack services have been stopped.</p> <p>This is an important step to avoid inconsistencies in the data migrated for the data-plane adoption procedure caused by resource changes after the DB has been copied to the new deployment.</p> <p>Some services are easy to stop because they only perform short asynchronous operations, but other services are a bit more complex to gracefully stop because they perform synchronous or long running operations that we may want to complete instead of aborting them.</p> <p>Since gracefully stopping all services is non-trivial and beyond the scope of this guide we'll proceed with the force method but present a couple of recommendations on how to check some things in the services.</p>"},{"location":"openstack/stop_openstack_services/#variables","title":"Variables","text":"<p>Define the shell variables used in the steps below. The values are just illustrative and refer to a single node standalone director deployment, use values that are correct for your environment:</p> <pre><code>CONTROLLER1_SSH=\"ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100\"\nCONTROLLER2_SSH=\"\"\nCONTROLLER3_SSH=\"\"\n</code></pre> <p>We chose to use these ssh variables with the ssh commands instead of using ansible to try to create instructions that are independent on where they are running, but ansible commands could be used to achieve the same result if we are in the right host, for example to stop a service:</p> <pre><code>. stackrc ansible -i $(which tripleo-ansible-inventory) Controller -m shell -a \"sudo systemctl stop tripleo_horizon.service\" -b\n</code></pre>"},{"location":"openstack/stop_openstack_services/#pre-checks","title":"Pre-checks","text":"<p>We can stop OpenStack services at any moment, but we may leave things in an undesired state, so at the very least we should have a look to confirm that there are no long running operations that require other services.</p> <p>Ensure that there are no ongoing instance live migrations, volume migrations (online or offline), volume creation, backup restore, attaching, detaching, etc.</p> <pre><code>openstack server list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|'\nopenstack volume list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|'| grep -vi error\nopenstack volume backup list --all-projects -c ID -c Status |grep -E '\\| .+ing \\|' | grep -vi error\nopenstack image list -c ID -c Status |grep -E '\\| .+ing \\|'\n</code></pre>"},{"location":"openstack/stop_openstack_services/#stopping-control-plane-services","title":"Stopping control plane services","text":"<p>We can stop OpenStack services at any moment, but we may leave things in an undesired state, so at the very least we should have a look to confirm that there are no ongoing  operations.</p> <p>1- Connect to all the controller nodes. 2- Stop the services. 3- Make sure all the services are stopped.</p> <p>The cinder-backup service on OSP 17.1 could be running as Active-Passive under pacemaker or as Active-Active, so we'll have to check how it's running and stop it.</p> <p>These steps can be automated with a simple script that relies on the previously defined environmental variables and function:</p> <pre><code># Update the services list to be stopped\nServicesToStop=(\"tripleo_horizon.service\"\n                \"tripleo_keystone.service\"\n                \"tripleo_cinder_api.service\"\n                \"tripleo_cinder_api_cron.service\"\n                \"tripleo_cinder_scheduler.service\"\n                \"tripleo_cinder_backup.service\"\n                \"tripleo_glance_api.service\"\n                \"tripleo_neutron_api.service\"\n                \"tripleo_nova_api.service\"\n                \"tripleo_placement_api.service\")\n\nPacemakerResourcesToStop=(\"openstack-cinder-volume\"\n                          \"openstack-cinder-backup\")\n\necho \"Stopping systemd OpenStack services\"\nfor service in ${ServicesToStop[*]}; do\n    for i in {1..3}; do\n        SSH_CMD=CONTROLLER${i}_SSH\n        if [ ! -z \"${!SSH_CMD}\" ]; then\n            echo \"Stopping the $service in controller $i\"\n            if ${!SSH_CMD} sudo systemctl is-active $service; then\n                ${!SSH_CMD} sudo systemctl stop $service\n            fi\n        fi\n    done\ndone\n\necho \"Checking systemd OpenStack services\"\nfor service in ${ServicesToStop[*]}; do\n    for i in {1..3}; do\n        SSH_CMD=CONTROLLER${i}_SSH\n        if [ ! -z \"${!SSH_CMD}\" ]; then\n            echo \"Checking status of $service in controller $i\"\n            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive &gt;/dev/null; then\n               echo \"ERROR: Service $service still running on controller $i\"\n            fi\n        fi\n    done\ndone\n\necho \"Stopping pacemaker OpenStack services\"\nfor i in {1..3}; do\n    SSH_CMD=CONTROLLER${i}_SSH\n    if [ ! -z \"${!SSH_CMD}\" ]; then\n        echo \"Using controller $i to run pacemaker commands\"\n        for resource in ${PacemakerResourcesToStop[*]}; do\n            if ${!SSH_CMD} sudo pcs resource config $resource; then\n                ${!SSH_CMD} sudo pcs resource disable $resource\n            fi\n        done\n        break\n    fi\ndone\n</code></pre>"},{"location":"openstack/troubleshooting/","title":"Troubleshooting","text":"<p>This document contains information about various issues you might face and how to solve them.</p>"},{"location":"openstack/troubleshooting/#errimagepull-due-to-missing-authentication","title":"ErrImagePull due to missing authentication","text":"<p>The deployed containers pull the images from private containers registries that can potentially return authentication errors like:</p> <pre><code>Failed to pull image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\":\nrpc error: code = Unknown desc = unable to retrieve auth token: invalid\nusername/password: unauthorized: Please login to the Red Hat Registry using\nyour Customer Portal credentials.\n</code></pre> <p>An example of a failed pod:</p> <pre><code>  Normal   Scheduled       3m40s                  default-scheduler  Successfully assigned openstack/rabbitmq-server-0 to worker0\n  Normal   AddedInterface  3m38s                  multus             Add eth0 [10.101.0.41/23] from ovn-kubernetes\n  Warning  Failed          2m16s (x6 over 3m38s)  kubelet            Error: ImagePullBackOff\n  Normal   Pulling         2m5s (x4 over 3m38s)   kubelet            Pulling image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\"\n  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Failed to pull image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\": rpc error: code  ... can be found here: https://access.redhat.com/RegistryAuthentication\n  Warning  Failed          2m5s (x4 over 3m38s)   kubelet            Error: ErrImagePull\n  Normal   BackOff         110s (x7 over 3m38s)   kubelet            Back-off pulling image \"registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.0\"\n</code></pre> <p>To solve this issue we need to get a valid pull-secret from the official Red Hat console site, store this pull secret locally in a machine with access to the Kubernetes API (service node), and then run:</p> <pre><code>oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=&lt;pull_secret_location.json&gt;\n</code></pre> <p>The previous command will make available the authentication information in all the cluster's compute nodes, then trigger a new pod deployment to pull the container image with:</p> <pre><code>kubectl delete pod rabbitmq-server-0 -n openstack\n</code></pre> <p>And the pod should be able to pull the image successfully.  For more information about what container registries requires what type of authentication, check the official docs.</p>"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 94e23a94b..80a3f7bcb 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -55,6 +55,11 @@
          <lastmod>2023-11-02</lastmod>
          <changefreq>daily</changefreq>
     </url>
+    <url>
+         <loc>https://openstack-k8s-operators.github.io/data-plane-adoption/openstack/heat_adoption/</loc>
+         <lastmod>2023-11-02</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
     <url>
          <loc>https://openstack-k8s-operators.github.io/data-plane-adoption/openstack/horizon_adoption/</loc>
          <lastmod>2023-11-02</lastmod>
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index f2bc02ea1..acb0028bb 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ