config.example.json

{
  "debug": true,
  "userAgent": "SSScraper",
  "htmlCache": {
    "directory": "./cache"
  },
  "pdfCache": {
    "directory": "./pdf-cache"
  },
  "request": {
    "timeoutInMs": 2000,
    "domainGlob": "**",
    "parallelism": 2,
    "delayInMs": 300,
    "randomDelayInMs": 1500
  },
  "input": {
    "startUrl": "https://gotripod.com/",
    "urlFilters": ["^https://gotripod.com/.*"],
    "disallowedUrlFilters": ["^https://gotripod.com/private/.*"]
  },
  "output": {
    "type": "jsonl",
    "filename": "documents.jsonl",
    "webhook": null
  },
  "html": {
    "selectors": {
      "id": "{{ .Request.URL }}",
      "title": "title",
      "statusCode": "{{ .Response.StatusCode }}",
      "content": "#content",
      "metaDescription": "meta[name=\"description\"]",
      "metaKeywords": "meta[name=\"keywords\"]",
      "h1": "h1",
      "h2": "h2",
      "url": "{{ .Request.URL }}",
      "section": "{{ .Request.URL.Path | regexFind \"/(insights|work)/\" | replace \"/\" \"\" }}",
      "urlPathSplit": "{{ .Request.URL.Path | splitList \"/\" | join \" \" }}"
    }
  },
  "pdf": {
    "enabled": true,
    "selectors": {
      "id": "{{ .Request.URL }}",
      "title": "{{ index .Meta \"title\" }}",
      "url": "{{ .Request.URL }}",
      "content": "{{ .TextContent }}",
      "section": "{{ .Request.URL.Path | regexFind \"/(insights|work)/\" | replace \"/\" \"\" }}"
    }
  }
}