KANBAN-490

.gitignore

@@ -12,8 +12,9 @@ package-lock.json
 /dist
 /build
 /src/dist
-/src/public/sitemap.xml
-/src/public/main-sitemap.xml
+/public/sitemap.xml
+/public/robots.txt
+/public/main-sitemap.xml
 /src/resourceDescriptors.js
 
 # IDEs and editors

bin/build_robots.js

@@ -0,0 +1,28 @@
+const fs = require('fs');
+const env = process.env.NODE_ENV;
+const directory = './public';
+const filename = 'robots.txt';
+
+const PROD_CONTENT = `User-agent: *
+Allow: /
+Disallow: /search
+Disallow: /alliancemine
+Disallow: /jbrowse
+Sitemap: https://www.alliancegenome.org/sitemap.xml
+Sitemap: https://www.alliancegenome.org/api/sitemap.xml`;
+
+const OTHER_CONTENT = `# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow: /
+`;
+
+const content = env === 'production' ? PROD_CONTENT : OTHER_CONTENT;
+
+console.log(`writing to... ${directory}/${filename}`)
+fs.writeFile(`${directory}/${filename}`, content, (err) => {
+  if (err) {
+    console.error('Error writing robots.txt file:', err);
+  } else {
+    console.log('robots.txt file written successfully');
+  }
+});

bin/build_sitemap.js

@@ -7,7 +7,7 @@ var sitemapindex = sm.buildSitemapIndex({
   ]
 });
 
-fs.writeFileSync('src/public/sitemap.xml', sitemapindex.toString());
+fs.writeFileSync('public/sitemap.xml', sitemapindex.toString());
 
 var sitemap = sm.createSitemap({
   hostname: 'https://www.alliancegenome.org',
@@ -29,4 +29,4 @@ var sitemap = sm.createSitemap({
   ]
 });
 
-fs.writeFileSync('src/public/main-sitemap.xml', sitemap.toString());
+fs.writeFileSync('public/main-sitemap.xml', sitemap.toString());

package.json

@@ -67,7 +67,8 @@
     "test": "react-scripts test",
     "eject": "react-scripts eject",
     "prebuild": "npm run resource-descriptors",
-    "postbuild": "npm run sitemap",
+    "postbuild": "npm run sitemap && npm run robots",
+    "robots": "node bin/build_robots.js",
     "sitemap": "node bin/build_sitemap.js",
     "resource-descriptors": "node bin/build_resource_descriptors.js"
   },