config.json.example

{
  "mongo-scrapy": {
    "host": "localhost",
    "mongo_port": 27017,
    "proxy_host": "",
    "proxy_port": 3128,
    "project": "hci",
    "queueCol": "crawler.queue",
    "pageStoreCol": "crawler.pages",
    "jobListCol": "crawler.jobs",
    "jobLogsCol": "logs",
    "scrapy_port" : 6800,
    "maxdepth" : 1,
    "download_delay" : 0.5,
    "max_simul_requests" : 36,
    "max_simul_requests_per_host" : 12
  },
  "memoryStructure": {
    "thrift.IP": "localhost",
    "thrift.port": 9090,
    "lucene.path": "/home/hci/lucene",
    "log.level": "INFO",
    "max_simul_pages_indexing": 100,
    "max_simul_links_indexing": 10000
  },
  "twisted" : {
    "port" : 6978
  },
  "precisionLimit": 2,
  "defaultWebEntityCreationRule": {
    "regexp": "(s:[a-zA-Z]+(\\|t:[0-9]+)?(\\|h:[^\\|]+)(\\|h:[^\\|]+)+)",
    "prefix": ""
  },
  "discoverPrefixes": ["http://bit.ly",
                       "http://t.co",
                       "http://tinyurl.com",
                       "http://goo.gl",
                       "http://fb.me",
                       "http://on.fb.me",
                       "http://ur1.ca",
                       "http://ow.ly"],
  "DEBUG": 0
}