Skip to content

Commit

Permalink
Merge branch 'monolithic_slit'
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sean-Morrison
Sean-Morrison committed Mar 27, 2024
2 parents d5edd5d + 68d5e0f commit 5a2889b
Show file tree
Hide file tree
Showing 13 changed files with 108 additions and 96 deletions.
6 changes: 3 additions & 3 deletions bin/dailylogger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ def send_email(subject, email_file, attachment, logger, content=None, from_domai
attachment = np.atleast_1d(attachment)
msg.preamble = 'You will not see this in a MIME-aware mail reader.\n'
for fa in attachment:
if ptt.exists(attachment):
with open(attachment, 'rb') as fp:
if ptt.exists(fa):
with open(fa, 'rb') as fp:
logdata = fp.read()
msg.add_attachment(logdata, maintype='text', subtype='plain', filename=ptt.basename(attachment))
msg.add_attachment(logdata, maintype='text', subtype='plain', filename=ptt.basename(fa))
s = smtplib.SMTP('localhost')
s.send_message(msg)
s.quit()
Expand Down
26 changes: 18 additions & 8 deletions bin/fieldmerge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -793,35 +793,45 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'),

if lite is True:
spAll_lite = spAll.copy()
spAll_lite.rename_column('DELTA_RA_LIST', 'DELTA_RA')
spAll_lite.rename_column('DELTA_DEC_LIST', 'DELTA_DEC')
for i in range(len(spAll)):
with warnings.catch_warnings():
warnings.filterwarnings(action='ignore', message='Mean of empty slice')
try:
spAll_lite[i]['ASSIGNED'] = str(min(np.array(spAll[i]['ASSIGNED'].split()).astype(int)))
spAll_lite[i]['ASSIGNED'] = str(min(np.array(spAll_lite[i]['ASSIGNED'].split()).astype(int)))
except:
spAll_lite[i]['ASSIGNED'] = '0'
try:
spAll_lite[i]['ON_TARGET'] = str(min(np.array(spAll[i]['ON_TARGET'].split()).astype(int)))
spAll_lite[i]['ON_TARGET'] = str(min(np.array(spAll_lite[i]['ON_TARGET'].split()).astype(int)))
except:
spAll_lite[i]['ON_TARGET'] = '0'
try:
spAll_lite[i]['VALID'] = str(min(np.array(spAll[i]['VALID'].split()).astype(int)))
spAll_lite[i]['VALID'] = str(min(np.array(spAll_lite[i]['VALID'].split()).astype(int)))
except:
spAll_lite[i]['VALID'] = '0'
try:
spAll_lite[i]['DECOLLIDED'] = str(min(np.array(spAll[i]['DECOLLIDED'].split()).astype(int)))
spAll_lite[i]['DECOLLIDED'] = str(min(np.array(spAll_lite[i]['DECOLLIDED'].split()).astype(int)))
except:
spAll_lite[i]['DECOLLIDED'] = '0'
try:
spAll_lite[i]['MOON_DIST'] = str(np.nanmean(np.array(spAll[i]['MOON_DIST'].split()).astype(float)))
spAll_lite[i]['MOON_DIST'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_DIST'].split()).astype(float)))
except:
spAll_lite[i]['MOON_DIST'] = 'nan'
try:
spAll_lite[i]['MOON_PHASE'] = str(np.nanmean(np.array(spAll[i]['MOON_PHASE'].split()).astype(float)))
spAll_lite[i]['MOON_PHASE'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_PHASE'].split()).astype(float)))
except:
spAll_lite[i]['MOON_PHASE'] = 'nan'
try:
spAll_lite[i]['CARTON_TO_TARGET_PK'] = str(int(np.array(spAll[i]['CARTON_TO_TARGET_PK'].split())[0]))
spAll_lite[i]['DELTA_RA'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_RA'].split()).astype(float)))
except:
spAll_lite[i]['DELTA_RA'] = 'nan'
try:
spAll_lite[i]['DELTA_DEC'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_DEC'].split()).astype(float)))
except:
spAll_lite[i]['DELTA_DEC'] = 'nan'
try:
spAll_lite[i]['CARTON_TO_TARGET_PK'] = str(int(np.array(spAll_lite[i]['CARTON_TO_TARGET_PK'].split())[0]))
except:
spAll_lite[i]['CARTON_TO_TARGET_PK'] = '0'

Expand All @@ -832,7 +842,7 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'),
for col in ['CARTON_TO_TARGET_PK']:
spAll_lite[col] = spAll_lite[col].astype(int)

for col in ['MOON_DIST','MOON_PHASE']:
for col in ['MOON_DIST','MOON_PHASE','DELTA_RA','DELTA_DEC']:
spAll_lite[col] = spAll_lite[col].astype(float)
else:
spAll_lite = None
Expand Down
17 changes: 13 additions & 4 deletions bin/readfibermaps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,13 @@ def readfibermaps(spplan2d=None, topdir=None, clobber=False, SOS=False, no_db=Fa
splog.open(logfile = ptt.join(SOS_opts['log_dir'], spFibermap.replace('.fits','.log')), append= (not clobber))

splog.info('readfibermaps started at '+ time.ctime())
try:
with fits.open(ptt.join(topdir, spFibermap), mode='update', checksum=True, output_verify="fix") as hdul:
hdul.verify('fix')
hdul.flush()
clobber = False
except:
clobber = True
if not clobber:
if (ptt.exists(ptt.join(topdir, spFibermap))):
try:
Expand Down Expand Up @@ -325,7 +332,7 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non
fibermap = readPlateplugMap(fibermap_file, fibermap, mjd, SOS=SOS, fast=fast,
exptime=exptime, plates=plates, legacy=legacy, no_db=no_db, indir=indir,
release=release, no_remote=no_remote, dr19=dr19)

fibermap['fiber_offset'] = 0
elif fps:
fibermap = read_table_yanny(fibermap_file, 'FIBERMAP')
fibermap.convert_bytestring_to_unicode()
Expand All @@ -339,6 +346,7 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non
dcol[dcol.data == -999] = np.NaN
fibermap = readFPSconfSummary(fibermap, mjd, sos=SOS, no_db = no_db, fast=fast,
release=release, no_remote=no_remote, dr19=dr19)
fibermap = flag_offset_fibers(fibermap)
fibermap['SCI_EXPTIME'] = np.NaN
hdr['CARTRIDGEID'] = 'FPS-S' if hdr['observatory'] == 'LCO' else 'FPS-N'

Expand All @@ -351,11 +359,11 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non
def flag_offset_fibers(fibermap):
splog.info('Flagging offset Fibers')
if not bool(int(fibermap.meta['is_dithered'])):
offsets = np.zeros(len(fibermap), dtype=bool)
fibermap.add_column(offsets,name='fiber_offset')
#offsets = np.zeros(len(fibermap), dtype=bool)
#fibermap.add_column(offsets,name='fiber_offset')
foff = fibermap['fiber_offset']

indx = np.where(np.logical_or((fibermap['DELTA_RA'].data != 0), (fibermap['DELTA_DEC'].data != 0)))[0]
indx = np.where(np.logical_or((fibermap['delta_ra'].data != 0), (fibermap['delta_dec'].data != 0)))[0]
foff[indx] = 1
return(fibermap)

Expand Down Expand Up @@ -1128,6 +1136,7 @@ def readPlateplugMap(plugfile, fibermap, mjd, SOS=False,
for i, row in enumerate(fibermap):
if 'bhm_spiders_clusters-efeds' in row['FIRSTCARTON']:
mag.data[i,1:] = mag.data[i,1:] - psffibercor
fibermap.add_column(Column('Plates', name = 'cadence'))
else:
programname = None

Expand Down
4 changes: 3 additions & 1 deletion bin/spSpec_reformat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import warnings
from PIL import Image, PngImagePlugin

mpl.use('Agg')

import time
import logging
from splog import Splog
Expand Down Expand Up @@ -698,7 +700,7 @@ def SDSS_specplot(basedir, Coadd_Table, spAll, catalogID, files = Table(), xra=[
parser.add_argument('--plot', '-p', action='store_true', help='Create spec plots')
parser.add_argument('--epoch', '-e', action='store_true', help='Run for epoch Coadds')
parser.add_argument('--lsdr10', action='store_true', help='Include Legacy Survey DR10 links on HTML')
parser.add_argument('--allsky', action='store_true', help='Include Legacy Survey DR10 links on HTML')
parser.add_argument('--allsky', action='store_true', help='Reformat for Allsky Custom Coadd')

args = parser.parse_args()
spSpec_reformat(args.topdir, args.run2d, args.run1d, args.field, args.mjd,
Expand Down
14 changes: 10 additions & 4 deletions bin/spplan_target.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,12 @@ def build_plan(spAll, use_catid=False, coadd_mjdstart = None):
plan = vstack([plan,new])

#if len(plan)> 400: break
if len(plan) == 0:
splog.info('No Valid TARGETS')
plan = None
else:
plan['FMJD_LIST']=plan['FMJD_LIST'].astype(str)
plan = mod_epoch(plan)
plan['FMJD_LIST']=plan['FMJD_LIST'].astype(str)

plan = mod_epoch(plan)
Expand Down Expand Up @@ -279,10 +285,10 @@ def CustomCoadd(name, topdir, run2d, run1d, cartons=None, catalogids=None, obs=N
if mjd is not None:
mjd_start = mjd_end = mjd
plan = build_plan(spAll, use_catid=use_catid, coadd_mjdstart = coadd_mjdstart)
write_plan(name, plan, topdir, run2d, run1d, mjdstart, mjdend,clobber=clobber,
rerun1d=rerun1d, use_catid=use_catid, obs=obs,
coadd_mjdstart=coadd_mjdstart)

if plan is not None:
write_plan(name, plan, topdir, run2d, run1d, mjdstart, mjdend,clobber=clobber,
rerun1d=rerun1d, use_catid=use_catid, obs=obs,
coadd_mjdstart=coadd_mjdstart)
splog.close()
return

Expand Down
2 changes: 2 additions & 0 deletions bin/uubatchpbs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ def build_cmd(topdir=None,run2d=None,run1d=None,idlutils_1d=None,
spSpecRef_key = "" # " --lsdr10"
flist_key = ""
fmerge_key = " --include_bad"
if run2d.lower() == 'master':
spSpecRef_key =" --lsdr10"

if epoch:
spSpecRef_key = spSpecRef_key + " --epoch"
Expand Down
13 changes: 9 additions & 4 deletions bin/uurundaily.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,9 @@ def get_key(fp):
if ptt.isdir(path.replace('?????', str(lastmjd))):
mjd.append(lastmjd)
else:
logger.info('skipping '+str(lastmjd)+' for '+mod+' obs='+obs)
send_email('skipping '+str(lastmjd)+' for '+mod+' obs='+obs,
ptt.join(getenv('HOME'), 'daily', 'etc','emails'), None, from_domain=from_domain)
ptt.join(getenv('HOME'), 'daily', 'etc','emails'), None, logger, from_domain=from_domain)
#email(subj = 'skipping '+str(lastmjd)+' for '+mod+' obs='+obs)
lastmjd = lastmjd - 1
if len(mjd) == 0:
Expand Down Expand Up @@ -191,8 +192,8 @@ def dailysummary(queue1, obs, run2d, run1d, module, logger, epoch = False, build
cores = 2
fmerge_cmd = ptt.join(getenv('HOME'),'daily','cmd','run_pyfieldmerge_epoch_{run2d}')

queue2.create(label = f"BOSS_Summary_{'-'.join(obs)}_{run2d}", nodes = 1, ppn = cores, walltime = "24:00:00",
alloc='sdss-np', qos = 'sdss', partition = 'sdss-np', mem_per_cpu = 64000, shared = True)
queue2.create(label = f"BOSS_Summary_{'-'.join(obs)}_{run2d}", nodes = 1, ppn = 64, walltime = "24:00:00",
alloc='sdss-np', qos = 'sdss', partition = 'sdss-np', mem_per_cpu = 7500, shared = False)

if not ptt.exists(fmerge_cmd):
makedirs(ptt.join(getenv('HOME'),'daily','cmd'),exist_ok=True)
Expand Down Expand Up @@ -234,7 +235,11 @@ def dailysummary(queue1, obs, run2d, run1d, module, logger, epoch = False, build
if percomp1 == 100 and percomppost == 100:
running=False
logger.info('exiting code')
return('Complete '+run2d +' MJD='+str(jdate) +' OBS='+','.join(obs),[fmerge_log+".o.log", fmerge_log+".e.log"] )
if build:
attachments = [fmerge_log+".o.log", fmerge_log+".e.log"]
else:
attachments = None
return('Complete '+run2d +' MJD='+str(jdate) +' OBS='+','.join(obs), attachments)#[fmerge_log+".o.log", fmerge_log+".e.log"] )
time.sleep(pause)
return (None, None)

Expand Down
38 changes: 19 additions & 19 deletions datamodel/fieldList_dm.par
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,13 @@ EXT1 PROGRAMNAME A "" "" "Program name within a give
EXT1 FIELDQUALITY A "" "" ""
EXT1 FIELDSN2 E "" "" "Overall (S/N)^2 for field; min of cameras"
EXT1 DEREDSN2 E "" "" "Overall Dereddened (S/N)^2 for field"
EXT1 QSURVEY K "" 0 "1 for an survey quality plate, 0 otherwise"
EXT1 QSURVEY K "" -1 "1 for an survey quality plate, 0 otherwise"
EXT1 MJDLIST A "" "" "List of MJD of each included exposures"
EXT1 NEXP K "" 0 "Number of Included Exposures"
EXT1 NEXP_B1 K "" 0 "Number of Included Exposures from b1"
EXT1 NEXP_R1 K "" 0 "Number of Included Exposures from r1"
EXT1 NEXP_B2 K "" 0 "Number of Included Exposures from b2"
EXT1 NEXP_R2 K "" 0 "Number of Included Exposures from r2"
EXT1 NEXP K "" -1 "Number of Included Exposures"
EXT1 NEXP_B1 K "" -1 "Number of Included Exposures from b1"
EXT1 NEXP_R1 K "" -1 "Number of Included Exposures from r1"
EXT1 NEXP_B2 K "" -1 "Number of Included Exposures from b2"
EXT1 NEXP_R2 K "" -1 "Number of Included Exposures from r2"
EXT1 EXPT_B1 E "" "" "Total Exposure Time of b1"
EXT1 EXPT_R1 E "" "" "Total Exposure Time of r1"
EXT1 EXPT_B2 E "" "" "Total Exposure Time of b2"
Expand Down Expand Up @@ -103,7 +103,7 @@ EXT1 GROFFGAL E "" "" "Spectrophoto offset for G-
EXT1 GRRMSGAL E "" "" "Spectrophoto RMS for G-R in galaxies"
EXT1 RIOFFGAL E "" "" "Spectrophoto offset for R-I in galaxies"
EXT1 RIRMSGAL E "" "" "Spectrophoto RMS for R-I in galaxies"
EXT1 NGUIDE K "" 0 "Number of guider frames during the exposures"
EXT1 NGUIDE K "" -999 "Number of guider frames during the exposures"
EXT1 SEEING20 E "arcsec" "" "Mean 20% seeing during exposures"
EXT1 SEEING50 E "arcsec" "" "Mean 50% seeing during exposures"
EXT1 SEEING80 E "" "" "Mean 80% seeing during exposures"
Expand All @@ -126,18 +126,18 @@ EXT1 SCHI2MAX E "" "" "Maximum skyChi2 over all e
EXT1 FBADPIX E "" "" "Fraction of bad pixels"
EXT1 FBADPIX1 E "" "" "Fraction of bad pixels from spectrograph #1"
EXT1 FBADPIX2 E "" "" "Fraction of bad pixels from spectrograph #2"
EXT1 N_TOTAL K "" 0 "Number of Sources"
EXT1 N_GALAXY K "" 0 "Number of Galaxies"
EXT1 N_QSO K "" 0 "Number of QSOs"
EXT1 N_STAR K "" 0 "Number of Stars"
EXT1 N_UNKNOWN K "" 0 "Number of Unknown Sources"
EXT1 N_SKY K "" 0 "Number of Skys"
EXT1 N_STD K "" 0 "Number of Standards"
EXT1 N_TARGET_QSO K "" 0 "Number of QSO Targeted"
EXT1 N_TARGET_MAIN K "" 0 "Number of Main Galaxy targeted"
EXT1 N_TARGET_LRG1 K "" 0 "Number of LRG1 targeted"
EXT1 N_TARGET_LRG2 K "" 0 "Number of LRG2 targeted"
EXT1 N_TARGET_ELG K "" 0 "Number of ELG targted"
EXT1 N_TOTAL K "" -999 "Number of Sources"
EXT1 N_GALAXY K "" -999 "Number of Galaxies"
EXT1 N_QSO K "" -999 "Number of QSOs"
EXT1 N_STAR K "" -999 "Number of Stars"
EXT1 N_UNKNOWN K "" -999 "Number of Unknown Sources"
EXT1 N_SKY K "" -999 "Number of Skys"
EXT1 N_STD K "" -999 "Number of Standards"
EXT1 N_TARGET_QSO K "" -999 "Number of QSO Targeted"
EXT1 N_TARGET_MAIN K "" -999 "Number of Main Galaxy targeted"
EXT1 N_TARGET_LRG1 K "" -999 "Number of LRG1 targeted"
EXT1 N_TARGET_LRG2 K "" -999 "Number of LRG2 targeted"
EXT1 N_TARGET_ELG K "" -999 "Number of ELG targted"
EXT1 SUCCESS_MAIN E "" "" "Success rate of Main Galaxy targets"
EXT1 SUCCESS_LRG1 E "" "" "Success rate of LRG1s"
EXT1 SUCCESS_LRG2 E "" "" "Success rate of LRG2s"
Expand Down
Loading

0 comments on commit 5a2889b

Please sign in to comment.