Google Trends functions

#This script automates the downloading of Google Trends.
#It works best with firefox in combination with the Tab Mix Plus add-on that is used to automate tab closing.
#Ask firefox not to prompt for new downloads and this script should run automatically.
#Google Trends restricts the number of download to roughly 400 at a time.

URL_GT=function(keyword="", country=NA, region=NA, year=NA, month=1, length=3){

  start="http://www.google.com/trends/trendsReport?hl=en-US&q="
  end="&cmpt=q&content=1&export=1"
  geo=""
  date=""

#Geographic restrictions
  if(!is.na(country)) {
    geo="&geo="
    geo=paste(geo, country, sep="")
    if(!is.na(region)) geo=paste(geo, "-", region, sep="")
  }

  queries=keyword[1]
  if(length(keyword)>1) {
    for(i in 2:length(keyword)){
    queries=paste(queries, "%2C ", keyword[i], sep="")
    }
  }

#Dates
  if(!is.na(year)){
    date="&date="
    date=paste(date, month, "%2F", year, " ", length, "m", sep="")
  }

  URL=paste(start, queries, geo, date, end, sep="")
  return(URL)
}

downloadGT=function(URL, downloadDir){

#Determine if download has been completed by comparing the number of files in the download directory to the starting number
  startingFiles=list.files(downloadDir)
  browseURL(URL)
  endingFiles=list.files(downloadDir)

  while(length(setdiff(endingFiles,startingFiles))==0) {
    Sys.sleep(3)
    endingFiles=list.files(downloadDir)
  }
  filePath=setdiff(endingFiles,startingFiles)
  return(filePath)
}


readGT=function(filePath){
rawFiles=list()

  for(i in 1:length(filePath)){
    if(length(filePath)==1) rawFiles[[1]]=read.csv(filePath, header=F, blank.lines.skip=F)
    if(length(filePath)>1) rawFiles[[i]]=read.csv(filePath[i], header=F, blank.lines.skip=F)
  }

  output=data.frame()
  name=vector()

  for(i in 1:length(rawFiles)){
    data=rawFiles[[i]]
    raw_name=as.character(data[1,1])
    raw_name=substr(raw_name, 22, nchar(raw_name))
  
#Create a vector called name containing the search terms
    if(grepl(";", raw_name)) {
      separators=gregexpr(";", raw_name)[[1]]
      separators=c(separators, nchar(raw_name)+1)
      
      name[1]=substr(raw_name, 1, separators[1]-1)
      for(j in 2:length(separators)) {
        name[j]=substr(raw_name, separators[j-1]+2, separators[j]-1)
      }
    } else {name=raw_name}
    
#Select the time series
    start=which(data[,1]=="")[1]+3
    stop=which(data[,1]=="")[2]-2
    
#Skip to next if file is empty
    if(ncol(data)<2) next
    if(is.na(which(data[,1]=="")[2]-2)) next
    
    data=data[start:stop,]
    data[,1]=as.character(data[,1])

#Convert all columns except date column into numeric
    for(j in 2:ncol(data)) data[,j]=as.numeric(as.character(data[,j]))

#FORMAT DATE
    len=nchar(data[1,1])

#Monthly data
    if(len==7) {
        data[,1]=as.Date(paste(data[,1], "-1", sep=""), "%Y-%m-%d")
        data[,1]=sapply(data[,1], seq, length=2, by="1 month")[2,]-1
        data[,1]=as.Date(data[,1], "%Y-%m-%d", origin="1970-01-01")
      }

#Weekly data
      if(len==23){
        data[,1]=sapply(data[,1], substr, start=14, stop=30)
        data[,1]=as.Date(data[,1], "%Y-%m-%d")
      }

#Daily data
      if(len==10) data[,1]=as.Date(data[,1], "%Y-%m-%d")
      
#Structure into panel data format
      panelData=data[1:2]
      panelData[3]=name[1]
      names(panelData)=c("Date", "SVI", "Keyword")
      if(ncol(data)>2) {

        for(j in 3:ncol(data)) {
          appendData=data[c(1,j)]
          appendData[3]=name[j-1]
          names(appendData)=c("Date", "SVI", "Keyword")
          panelData=rbind(panelData, appendData)
        }
      }

#Add file name  
      panelData[ncol(panelData)+1]=filePath[i]
  
#Add path to filename
      names(panelData)[4]="Path"
      
#Merge several several files into one
      if(i==1) output=panelData
      if(i>1) output=rbind(output, panelData)
  }
  return(output)
}

readGeoGT=function(filePath){
  output=data.frame()
  rawFiles=list()
  for(i in 1:length(filePath)){
    if(length(filePath)==1) rawFiles[[1]]=read.csv(filePath, header=F, blank.lines.skip=F)
    if(length(filePath)>1) rawFiles[[i]]=read.csv(filePath[i], header=F, blank.lines.skip=F)
  }

  for(i in 1:length(rawFiles)){
    data=rawFiles[[i]]
    start=which(data[,1]=="")[3]+3
    stop=which(data[,1]=="")[4]-1
    names=data[start-1,]
    
    for(j in 1:ncol(names)) names(data)[j]=as.character(names[1,j])
      data=data[start:stop,]
      data[,1]=as.character(data[,1])
      data[,-1]=as.numeric(as.character(data[,-1]))
      data[ncol(data)+1]=filePath[i]
      
      output=rbind(output, data)
    }
  return(output)
}