C_sf_load_single_LC_detail.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A modification of  \n",
    "# C_sf_load_single_LC.py  \n",
    "# that was routinely used to create \n",
    "# master files.\n",
    "\n",
    "\n",
    "# Only for quasars in  a specified magnitude range, we \n",
    "# want to make 'detail' master files,  with  \n",
    "# all epoch1,  epoch2  data per row , \n",
    "# apart from just  delta_mag,  delta_time,  delta_err..\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Workflow : \n",
    "\n",
    "1) Decide which QSO / stars need to be considered \n",
    "\n",
    "2) Create 'detailed master files' with  delta_time,  delta_mag ,  err(delta_mag),  t1, t2,  m1,  m2 , e1, e2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np \n",
    "import sys\n",
    "\n",
    "from CRTS_paper_modules import update_progress  as upd \n",
    "import CRTS_paper_modules as mod"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) Select QSO  or Stars Sample "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zipping CRTS-SDSS quasars catalog from ../data_products/CRTS_SDSS_catalogs/CRTS_SDSS_cross_matched_qso_DB_QSO_catalog.txt\n",
      "Read in 7601 quasars from CRTS\n",
      "zipping CRTS-SDSS stars catalog...\n",
      "Read in catalog for 48250 stars from CRTS \n"
     ]
    }
   ],
   "source": [
    "\n",
    "cols1,  qso_cat = mod.get_qso_catalog() \n",
    "cols2 , star_cat= mod.get_stars_catalog()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using now only lightcurves with SDSS  18.000000< r < 18.500000\n",
      "\n",
      "Choosing stars with  SDSS   -1.00<g-i<3.00\n",
      " These cuts reduced the number of stars  in the sample from 48250 to 3487\n",
      "\n",
      " Returning only QSO with  an SDSS counterpart within 2.000000 arcsec\n",
      "\n",
      " These cuts reduced the number of qso  in the sample from 7601 to 333\n",
      "\n",
      "Using now only lightcurves with SDSS  18.500000< r < 19.000000\n",
      "\n",
      "Choosing stars with  SDSS   -1.00<g-i<3.00\n",
      " These cuts reduced the number of stars  in the sample from 48250 to 3825\n",
      "\n",
      " Returning only QSO with  an SDSS counterpart within 2.000000 arcsec\n",
      "\n",
      " These cuts reduced the number of qso  in the sample from 7601 to 747\n"
     ]
    }
   ],
   "source": [
    "Min_arr = [ 18,   18.5 ]\n",
    "Max_arr = [ 18.5, 19   ]\n",
    "\n",
    "qso_names = {}   # need to initialize the dict beforehand !!! \n",
    "star_names = {}   # need to initialize the dict beforehand !!! \n",
    "\n",
    "mag = 'r'   # which magnitudes to use for cutting and reporting \n",
    "        \n",
    "for i in range(len(Min_arr)):\n",
    "    \n",
    "    Min = Min_arr[i]\n",
    "    Max = Max_arr[i]\n",
    "\n",
    "    print('\\nUsing now only lightcurves with SDSS  %f< %s < %f' % (Min, mag, Max))\n",
    "\n",
    "    # having gi from -1 to 3  includes both blue and red stars \n",
    "    good_ids_stars = mod.cut_stars(star_cat = star_cat,mMin = Min, mMax=Max, mErrMax = 0.3, gi_Min = -1,\n",
    "                                          gi_Max=3, cut_mag=mag + '_mMed')\n",
    "   \n",
    "    good_ids_QSO = mod.cut_qso(qso_cat=qso_cat, mMin = Min, mMax=Max, mErrMax = 0.3, \n",
    "                           cut_mag=mag)\n",
    "\n",
    "    qso_names[Min] = good_ids_QSO\n",
    "    star_names[Min]   = good_ids_stars\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2)  Create detailed master files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing detailed master files for stars, in range 18.00-18.50\n",
      "[######### ] 99.00%\n",
      "All in all, we saved 3487 SF files \n",
      "Preparing detailed master files for stars, in range 18.50-19.00\n",
      "[######### ] 99.00%\n",
      "All in all, we saved 3825 SF files \n"
     ]
    }
   ],
   "source": [
    "\n",
    "choice = 'stars'  # 'qso'\n",
    "survey  = 'CRTS'\n",
    "\n",
    "inDir =  '../proc_LC_'+survey+'/'+choice+'/'\n",
    "outDir = '../data_products/sf_file_per_LC/' +choice+'_detailed/'\n",
    "\n",
    "for m in range(len(Min_arr)) : \n",
    "    Min = Min_arr[m]\n",
    "    Max = Max_arr[m]\n",
    "\n",
    "    \n",
    "    if choice == 'qso' : \n",
    "        names = qso_names[Min]\n",
    "        \n",
    "    if choice == 'stars':\n",
    "        names = star_names[Min]\n",
    "        \n",
    "    total = float(len(names))\n",
    "    count = 0\n",
    "    \n",
    "    print('Preparing detailed master files for %s, in range %.2f-%.2f'%(choice, Min, Max))\n",
    "    \n",
    "    for i in range(len(names)): \n",
    "\n",
    "        percent = 100*(count / total)\n",
    "        if (count % 10) == 0 : # every tenth loop.. \n",
    "            upd(int(percent)) # print progress bar  \n",
    "        count += 1\n",
    "        \n",
    "        if choice == 'qso':\n",
    "            name  = 'out_'+names[i] +'.txt'\n",
    "            \n",
    "        if choice == 'stars' : \n",
    "            name = 'out_' + names[i]+'.dat.txt'\n",
    "            \n",
    "        # load the mjd, flux, and flux_error  from the read file \n",
    "        mjd,flx,err = np.loadtxt(inDir+name,usecols=(0,1,2),unpack=True)\n",
    "\n",
    "        # check for any nan's...\n",
    "        add = np.where(np.isnan(flx) == True)[0]\n",
    "\n",
    "        # Delete all rows with nan's - there should be none in files I provide.. \n",
    "        mjd = np.delete(mjd,add); flx = np.delete(flx,add); err = np.delete(err,add)\n",
    "\n",
    "        # Sort according to mjd's \n",
    "        ind = np.argsort(mjd)\n",
    "        mjd = mjd[ind]; flx = flx[ind]; err = err[ind]\n",
    "\n",
    "        # Calculate tau, mag difference and err difference for each pair (k,j), \n",
    "        # where tau=t_j-t_k (j>k) \n",
    "\n",
    "        # declare a priori the storage to make more efficient - \n",
    "        # appending is not very quick ... \n",
    "        N = np.sum(np.arange(2, len(mjd)+1)-1)\n",
    "\n",
    "        delflx = np.zeros(N);  delflxerr = np.zeros(N); tau = np.zeros(N)\n",
    "        t1 = np.zeros(N) ;   t2 =np.zeros(N);     m1 =np.zeros(N);  m2 =np.zeros(N)\n",
    "        e1=np.zeros(N); e2   =np.zeros(N)\n",
    "\n",
    "        pos = 0 \n",
    "        for j in range(len(mjd)-1):\n",
    "            for k in range(j+1):     \n",
    "                tau[pos] = mjd[j+1]-mjd[k]   # \n",
    "                delflx[pos] = flx[j+1] - flx[k]\n",
    "                delflxerr[pos] = np.sqrt(err[k]**2+err[j+1]**2)\n",
    "                t1[pos] = mjd[k] ; t2[pos] = mjd[j+1]\n",
    "                m1[pos] = flx[k] ; m2[pos] = flx[j+1]\n",
    "                e1[pos] = err[k] ; e2[pos] = err[j+1]\n",
    "\n",
    "                pos  += 1 \n",
    "\n",
    "        # sort according to tau\n",
    "        #int0 = np.argsort(tau)\n",
    "        #tau = tau[int0]; delflx = delflx[int0]\n",
    "        #delflxerr = delflxerr[int0]\n",
    "\n",
    "\n",
    "        ##### FILE SAVING #######\n",
    "        DATA = np.column_stack((delflx,tau, delflxerr, t1, t2, m1, m2, e1, e2))    \n",
    "        outfile = outDir + 'SF_' + names[i] + '.txt'\n",
    "        np.savetxt(outfile, DATA, delimiter =' ', fmt=\"%s\") \n",
    "\n",
    "    print('\\nAll in all, we saved %d SF files ' % (count))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}