crawl.conf

[crawler]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Configuration for this specific instance of the crawler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; the crawler ID is a numeric identifier used to identify a single crawler
;; thread, and must be both unique and used contiguously.
;; if you specify id=1 and thread=8, then this instance of crawld
;; would launch eight threads, identifying themselves as crawler 1, crawler 2,
;; and so on up to crawler 8. you would then specify crawler=9 in the next
;; instance that you configure, and so on.
;id=1

;; Specify the number of threads in this instance
;threads=1

;; if crawling should happen verbosely, set this to 1
; verbose=yes

;; Whether to fork into the background or not
; detach=yes

[cluster]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Cluster configuration
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; The environment is used in logging to distinguish groups of instances
;; from one another. For dynamic clustering, it's also used to generate
;; the etcd key paths, so that members in different environments using the
;; same etcd cluster don't interfere with one another.
;environment=production

;; Dynamic clustering will use etcd if configured to do so, in which case
;; the crawler ID specified above will be ignored. Both a name (used to
;; create the etcd key path) and registry URL must be specified
;name=anansi
;registry=http://localhost:2379/

;; Static clustering is used if dynamic clustering is not configured.
;; The total number of crawl threads across the cluster should be configured
;; here (and the same value across all members of the cluster)
;threads=1

[queue]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Queue configuration
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
name=db
;; if using the 'db' queue module, specify a database connection URI
uri=mysql://root@localhost/anansi
;; set to true to enable query debugging
debug-queries=no
;; set to true to enable error debugging
debug-errors=no

[processor]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Processor configuration
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; The processor module is responsible for parsing retrieved resources
;; and adding new links to the crawl queue as a result. Some processor
;; modules may apply specific policies to the process, rejecting certain
;; resources based upon pre-defined criteria.

;; The 'rdf' processor parses resources as RDF and adds any URIs found in
;; the graphs to the crawl queue.
name=rdf

;; The 'lod' processor is an extension of the RDF processor which also
;; performs licensing checks upon the resources, rejecting any which aren't
;; explicitly licensed as open data.
; name=lod

;; The LOD processor must be configured with predicates and a licensing
;; white and/or blacklist to be useful.
[lod:licenses]
predicate="http://purl.org/dc/terms/rights"
predicate="http://purl.org/dc/terms/license"
predicate="http://purl.org/dc/terms/accessRights"
predicate="http://creativecommons.org/ns#license"
predicate="http://www.w3.org/1999/xhtml/vocab#license"
whitelist="http://creativecommons.org/publicdomain/zero/1.0/"

[cache]
;; specify the location of the cache
uri=/var/spool/anansi
; uri=s3://anansi/
; username=user
; password=pass
; endpoint=s3.amazonaws.com

[log]
;; set stderr=1 to log output to standard error, not just syslog
;stderr=1
;; set the logging threshold level. valid values are emerg, alert, crit, err,
;; warn, notice, info, or debug.
;level=notice
;; set the logging facility, valid values include user, local0..local7,
;; although others may be valid on your system.
;level=daemon

;; Policy sections
[policy:content-types]
; blacklist=text/plain,application/x-unknown
; whitelist=text/html

[policy:schemes]
; whitelist=http
; blacklist=scp,mailto