WIP create product engineer job board

.gitignore

@@ -30,4 +30,5 @@ yarn.lock
 src/components/Layout/Fonts.scss
 .vercel
 .env
-*Type.ts
+*Type.ts
+.env.development

contents/handbook/engineering/posthog-com/cool-tech-jobs.mdx

@@ -0,0 +1,48 @@
+---
+title: Managing cool tech jobs
+---
+
+### Create a company/jobs:
+
+- Login to PostHog.com as a moderator
+- Navigate to `/cool-tech-jobs`
+- Click “Add a company”
+- Fill out the fields in the side modal
+- Click Create company
+
+When a company is created, its jobs are automatically scraped based on the job board URL/slug provided. If no jobs are found, the company doesn’t appear on `/cool-tech-jobs` (unless you’re a moderator, in which case it will appear semi-transparent).
+
+### Edit a company
+
+- Login to PostHog.com as a moderator
+- Navigate to `/cool-tech-jobs`
+- Click “Edit” under the desired company
+- Fill out the fields in the side modal
+- Click Update company
+
+Jobs will be re-scraped when a company is edited.
+
+### Delete a company
+
+- Login to PostHog.com as a moderator
+- Navigate to `/cool-tech-jobs`
+- Click “Delete” under the desired company
+- Confirm that you want to delete that company
+
+All jobs associated with the deleted company will be deleted along with the original company record.
+
+### Company fields
+
+- **Company name**
+- **Company website URL** - Used for the “Learn more” link
+- **Job board type** (Ashby, Greenhouse, Other) - If “Other” is selected, a custom scraper will need to be built. When the company is published, it will be hidden as no jobs will be scraped.
+- **Job board URL** (if Job board type is set to “Other”) - we’ll use this to build a custom scraper
+- **Job board slug** (if Job board type is *not* “Other”) - the job board’s slug. This is automatically created as you type the company name, as these usually mirror each other. Must be unique and is checked for uniqueness as you type.
+- **Company perks**
+- **Company logos** - SVG/PNG only
+
+Unless required conditionally (job board URL/slug), every company field is required.
+
+### Scraping
+
+Jobs are scraped hourly based on the provided job board URL/slug. Jobs are individually checked for freshness hourly. If a job URL 404s, it is deleted.

gatsby-node.ts

@@ -1,5 +1,6 @@
 import path from 'path'
 import { GatsbyNode } from 'gatsby'
+const axios = require('axios')
 
 export { createPages } from './gatsby/createPages'
 export { onCreateNode, onPreInit } from './gatsby/onCreateNode'
@@ -42,3 +43,24 @@ export const onCreateWebpackConfig: GatsbyNode['onCreateWebpackConfig'] = ({ sta
         },
     })
 }
+
+exports.createPages = async ({ actions }) => {
+    const { createPage } = actions
+
+    try {
+        const response = await axios.get('https://jobs.ashbyhq.com/supabase')
+        const jobData = JSON.parse(response.data)
+        const jobs = jobData?.jobBoard?.jobPostings || []
+
+        // Create the jobs page with the data
+        createPage({
+            path: '/jobs',
+            component: require.resolve('./src/templates/jobs.tsx'),
+            context: {
+                jobs: jobs,
+            },
+        })
+    } catch (error) {
+        console.error('Error fetching jobs:', error)
+    }
+}

package.json

@@ -148,6 +148,7 @@
         "remove-markdown": "^0.5.5",
         "request": "^2.88.2",
         "sass": "^1.43.2",
+        "scrapingbee": "^1.7.5",
         "slugify": "^1.6.0",
         "styled-components": "^5.3.3",
         "svg-sprite": "^1.5.0",

src/api/scrape.ts

@@ -0,0 +1,140 @@
+import axios from 'axios'
+
+// Helper function to get company logo URL
+async function getCompanyLogo(company: string): Promise<string | null> {
+    try {
+        // Try to get the company's homepage
+        const response = await axios.get(`https://${company}.com`)
+        const baseUrl = `https://${company}.com`
+
+        // Try favicon patterns first
+        const faviconPatterns = [
+            /<link[^>]*rel="icon"[^>]*href="([^"]*)"/,
+            /<link[^>]*rel="shortcut icon"[^>]*href="([^"]*)"/,
+            /<link[^>]*rel="favicon"[^>]*href="([^"]*)"/,
+        ]
+
+        // Helper function to resolve relative URLs
+        const resolveUrl = (path: string): string => {
+            if (path.startsWith('http')) return path
+            if (path.startsWith('//')) return 'https:' + path
+            if (path.startsWith('/')) return baseUrl + path
+            return baseUrl + '/' + path
+        }
+
+        // Try favicons
+        for (const pattern of faviconPatterns) {
+            const match = response.data.match(pattern)
+            if (match && match[1]) {
+                const faviconUrl = resolveUrl(match[1])
+                console.log(`Found favicon for ${company}:`, faviconUrl)
+
+                // Verify the favicon URL is accessible
+                try {
+                    await axios.head(faviconUrl)
+                    return faviconUrl
+                } catch (e) {
+                    console.log(`Favicon not accessible for ${company}, trying next option`)
+                    continue
+                }
+            }
+        }
+
+        // If no favicon found or accessible, try OpenGraph image
+        const ogImageMatch = response.data.match(/<meta[^>]*property="og:image"[^>]*content="([^"]*)"/)
+        if (ogImageMatch && ogImageMatch[1]) {
+            const ogImageUrl = resolveUrl(ogImageMatch[1])
+            console.log(`Found OG image for ${company}:`, ogImageUrl)
+
+            // Verify the OG image URL is accessible
+            try {
+                await axios.head(ogImageUrl)
+                return ogImageUrl
+            } catch (e) {
+                console.log(`OG image not accessible for ${company}, trying next option`)
+            }
+        }
+
+        // Try direct favicon.ico as last resort
+        const directFaviconUrl = `${baseUrl}/favicon.ico`
+        try {
+            await axios.head(directFaviconUrl)
+            console.log(`Found direct favicon.ico for ${company}:`, directFaviconUrl)
+            return directFaviconUrl
+        } catch (error) {
+            console.log(`No direct favicon.ico found for ${company}`)
+        }
+
+        console.log(`No logo found for ${company}`)
+        return null
+    } catch (error) {
+        console.error(`Error fetching logo for ${company}:`, error)
+        return null
+    }
+}
+
+/**
+ * @param {import('next').NextApiRequest} req
+ * @param {import('next').NextApiResponse} res
+ */
+export default async function scrape(req, res) {
+    if (req.method !== 'GET') {
+        res.setHeader('Allow', 'GET')
+        res.status(405).json({ success: false, message: 'Method not allowed' })
+        return
+    }
+
+    const company = req.query.company || 'supabase'
+
+    try {
+        // Get jobs data
+        const response = await axios.get(`https://jobs.ashbyhq.com/${company}`)
+        const match = response.data.match(/window\.__appData\s*=\s*({[^;]+})/)
+        if (!match) {
+            throw new Error(`Could not find job data in response for ${company}`)
+        }
+
+        const jobData = JSON.parse(match[1])
+        const jobs = jobData?.jobBoard?.jobPostings || []
+
+        // Try to get company logo
+        const logo = await getCompanyLogo(company)
+
+        const jobsWithLogo = jobs.map((job) => ({
+            ...job,
+            company,
+            logo,
+            link: `https://jobs.ashbyhq.com/${company}/${job.id}`,
+        }))
+
+        res.status(200).json({
+            success: true,
+            jobs: jobsWithLogo,
+        })
+    } catch (error) {
+        console.error(`API Error for ${company}:`, error)
+        res.status(500).json({
+            success: false,
+            message: `Error scraping jobs for ${company}`,
+            details: error.message,
+        })
+    }
+}
+
+const ATS_PATTERNS = [
+    {
+        name: 'Greenhouse',
+        pattern: /greenhouse\.io/i,
+        boardPattern: /greenhouse\.io\/[^/]+$/i,
+    },
+    {
+        name: 'Lever',
+        pattern: /lever\.co/i,
+        boardPattern: /lever\.co\/[^/]+$/i,
+    },
+    {
+        name: 'Workday',
+        pattern: /myworkdayjobs\.com/i,
+        boardPattern: /myworkdayjobs\.com\/[^/]+$/i,
+    },
+]

src/api/test-ashby.ts

@@ -0,0 +1,49 @@
+import { NextApiRequest, NextApiResponse } from 'next'
+import { findNewAshbyJobs, addNewCompaniesToConfig } from '../components/Jobs/jobsAshby'
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+    const logs: string[] = []
+
+    // Capture console.log output
+    const originalLog = console.log
+    console.log = (...args) => {
+        logs.push(args.map((arg) => (typeof arg === 'object' ? JSON.stringify(arg) : arg)).join(' '))
+        originalLog.apply(console, args)
+    }
+
+    try {
+        const companies = await findNewAshbyJobs()
+        const addedCompanies = await addNewCompaniesToConfig()
+
+        // Restore original console.log
+        console.log = originalLog
+
+        res.status(200).json({
+            debug: {
+                nodeEnv: process.env.NODE_ENV,
+                hasApiKey: !!process.env.GATSBY_SCRAPING_BEE_API_KEY,
+                apiKeyFirstChars: process.env.GATSBY_SCRAPING_BEE_API_KEY
+                    ? process.env.GATSBY_SCRAPING_BEE_API_KEY.substring(0, 4) + '...'
+                    : 'none',
+                logs: logs,
+            },
+            companiesFound: companies,
+            companiesAdded: addedCompanies,
+        })
+    } catch (error) {
+        // Restore original console.log
+        console.log = originalLog
+
+        res.status(500).json({
+            error: 'Failed to fetch companies',
+            details: error.message,
+            debug: {
+                nodeEnv: process.env.NODE_ENV,
+                hasApiKey: !!process.env.GATSBY_SCRAPING_BEE_API_KEY,
+                logs: logs,
+                errorMessage: error.message,
+                errorStack: error.stack,
+            },
+        })
+    }
+}

src/components/AskMax/index.tsx

@@ -15,7 +15,13 @@ interface AskMaxProps {
     children?: React.ReactNode
 }
 
-export default function AskMax({ border = false, className = '', quickQuestions, linkOnly = false, children }: AskMaxProps) {
+export default function AskMax({
+    border = false,
+    className = '',
+    quickQuestions,
+    linkOnly = false,
+    children,
+}: AskMaxProps) {
     const posthog = usePostHog()
     const { compact } = useLayoutData()
     const { openChat, setQuickQuestions } = useChat()

src/components/Docs/Intro.js

@@ -2,29 +2,40 @@ import React from 'react'
 import { CallToAction } from '../CallToAction'
 import CloudinaryImage from '../CloudinaryImage'
 
-export default function Intro({ subheader, title, description, buttonText, buttonLink, imageColumnClasses, imageUrl, imageClasses}) {
-  return (
-  <div className="bg-accent dark:bg-accent-dark border border-light dark:border-dark rounded flex flex-col items-center md:flex-row md:gap-4 mb-8">
-      <div className="p-4 pb-0 md:p-8 flex-1 w-full">
-          <p className="text-[15px] text-primary/60 dark:text-primary-dark/75 mb-1">{subheader}</p>
-          <h1 className="text-3xl md:text-4xl mt-0 mb-1 md:mb-2">{title}</h1>
-          <h3 className="text-base md:text-lg font-semibold text-primary/60 dark:text-primary-dark/75 !leading-tight"> {description}</h3>
-          <CallToAction to={buttonLink}>{buttonText}</CallToAction>
-      </div>
+export default function Intro({
+    subheader,
+    title,
+    description,
+    buttonText,
+    buttonLink,
+    imageColumnClasses,
+    imageUrl,
+    imageClasses,
+}) {
+    return (
+        <div className="bg-accent dark:bg-accent-dark border border-light dark:border-dark rounded flex flex-col items-center md:flex-row md:gap-4 mb-8">
+            <div className="p-4 pb-0 md:p-8 flex-1 w-full">
+                <p className="text-[15px] text-primary/60 dark:text-primary-dark/75 mb-1">{subheader}</p>
+                <h1 className="text-3xl md:text-4xl mt-0 mb-1 md:mb-2">{title}</h1>
+                <h3 className="text-base md:text-lg font-semibold text-primary/60 dark:text-primary-dark/75 !leading-tight">
+                    {' '}
+                    {description}
+                </h3>
+                <CallToAction to={buttonLink}>{buttonText}</CallToAction>
+            </div>
 
-      {imageUrl && (
-              <figure className="m-0 mt-auto p-0 md:pr-8 flex items-end">
-                  <CloudinaryImage
-                      alt=""
-                      placeholder="none"
-                      quality={100}
-                      className={imageColumnClasses}
-                      imgClassName={imageClasses}
-                      src={imageUrl}
-                  />
-              </figure>
-          )}
-
-  </div>
-)
+            {imageUrl && (
+                <figure className="m-0 mt-auto p-0 md:pr-8 flex items-end">
+                    <CloudinaryImage
+                        alt=""
+                        placeholder="none"
+                        quality={100}
+                        className={imageColumnClasses}
+                        imgClassName={imageClasses}
+                        src={imageUrl}
+                    />
+                </figure>
+            )}
+        </div>
+    )
 }

src/components/Docs/ResourceItem.js

@@ -13,13 +13,13 @@ export default function ResourceItem({ title, description, Image, gatsbyImage, u
                         </h6>
                     )}
                     <h4 className="m-0 text-lg leading-tight text-primary dark:text-primary-dark">{title}</h4>
-                    <p className="text-primary/60 dark:text-primary-dark/60 text-[15px] mt-1 mb-0 !leading-tight">{description}</p>
+                    <p className="text-primary/60 dark:text-primary-dark/60 text-[15px] mt-1 mb-0 !leading-tight">
+                        {description}
+                    </p>
                 </div>
                 {Image && (
                     <div className="flex justify-end w-full h-24">
-                        <div className="w-48 h-24 md:absolute bottom-0">
-                            {Image}
-                        </div>
+                        <div className="w-48 h-24 md:absolute bottom-0">{Image}</div>
                     </div>
                 )}
             </Link>

src/components/Footer/Footer.tsx

@@ -193,6 +193,10 @@ const linklist: IProps[] = [
                 title: 'PostHog on GitHub',
                 url: 'https://github.com/PostHog/posthog',
             },
+            {
+                title: 'Cool tech jobs',
+                url: '/cool-tech-jobs',
+            },
         ],
     },
     {