major refactor of select_mock_targets

.travis.yml

@@ -46,10 +46,10 @@ env:
         - ASTROPY_VERSION=1.2.1
         - SPHINX_VERSION=1.5
         - DESIUTIL_VERSION=1.9.9
-        - DESISPEC_VERSION=0.11.0
-        - DESISIM_VERSION=master
-        - DESIMODEL_VERSION=0.7.0
-        - DESIMODEL_DATA=branches/test-0.7.0
+        - DESISPEC_VERSION=0.18.0
+        - DESISIM_VERSION=0.25.1
+        - DESIMODEL_VERSION=0.9.2
+        - DESIMODEL_DATA=branches/test-0.9
         - SPECLITE_VERSION=0.5
         - MAIN_CMD='python setup.py'
         # Additional conda channels to use.

bin/mpi_select_mock_targets

@@ -19,7 +19,6 @@ parser.add_argument('--nside', help='Divide the DESI footprint into this healpix
 parser.add_argument('--tiles', help='Path to file with tiles to cover', type=str)
 parser.add_argument('--npix', help='Number of healpix per process', type=int, default=1)
 parser.add_argument('--healpixels', help='Integer list of healpix pixels (corresponding to nside) to process.', type=int, nargs='*', default=None)
-parser.add_argument('--realtargets', '-r', help='Path to real target catalog', type=str)
 parser.add_argument('-v','--verbose', action='store_true', help='Enable verbose output.')
 parser.add_argument('--no-check-env', action='store_true', help="Don't check NERSC environment variables")
 parser.add_argument('--sort-pixels', action='store_true', help="Sort pixels by galactic latitude")
@@ -162,9 +161,9 @@ if len(rankpix) > 0:
                 log.info('  nproc      {}'.format(args.nproc))
                 log.info('  verbose    {}'.format(args.verbose))
                 log.info('  healpixels {}'.format(rankpix[i:i+n]))
-                targets_truth(params, args.output_dir, seed=rankseeds[i], nside=args.nside,
-                            nproc=args.nproc, verbose=args.verbose,
-                            healpixels=rankpix[i:i+n])
+                targets_truth(params, output_dir=args.output_dir, seed=rankseeds[i],
+                              nside=args.nside, nproc=args.nproc, verbose=args.verbose,
+                              healpixels=rankpix[i:i+n])
 
             runtime = (time.time()-t0) / 60
             log.info('Pixels {} took {:.1f} minutes'.format(rankpix[i:i+n], runtime))

bin/select_mock_targets

@@ -12,7 +12,7 @@ import yaml
 import healpy as hp
 from astropy.table import Table
 
-from desitarget.mock.build import targets_truth, targets_truth_no_spectra
+from desitarget.mock.build import targets_truth
 from desiutil.log import get_logger, DEBUG
 import desimodel
 import desitarget.mock.io as mockio
@@ -25,14 +25,14 @@ parser.add_argument('-c', '--config', default='input.yaml')
 parser.add_argument('-O', '--output_dir', help='Path to write the outputs', type=str, default='./')
 parser.add_argument('-s', '--seed', help='Seed for random number generation', type=int, default=None)
 parser.add_argument('-n', '--nproc', type=int, help='number of concurrent processes to use [{}]'.format(nproc), default=nproc)
-parser.add_argument('--nside', help='Divide the DESI footprint into this healpix resolution', type=int, default=64)
+parser.add_argument('--nside', help='Divide the DESI footprint into this healpix resolution', type=int, default=None)
 parser.add_argument('--tiles', help='Path to file with tiles to cover', type=str)
 parser.add_argument('--healpixels', help='Integer list of healpix pixels (corresponding to nside) to process.', type=int, nargs='*', default=None)
-parser.add_argument('--realtargets', '-r', help='Path to real target catalog', type=str)
 parser.add_argument('--join', action='store_true', help='Join the target and truth files in output_dir.')
 parser.add_argument('--qa', action='store_true', help='Generate QA plots from the joined target and truth files in output_dir.')
 parser.add_argument('-v','--verbose', action='store_true', help='Enable verbose output.')
-parser.add_argument('--no-spectra', action='store_true')
+parser.add_argument('--no-spectra', action='store_true', help='Do not generate spectra.')
+parser.add_argument('--overwrite', action='store_true', help='Overwrite existing files.')
 
 args = parser.parse_args()
 
@@ -43,6 +43,10 @@ else:
 
 log.info('Starting select_mock_targets at {}'.format(time.asctime()))
 
+if args.nside is None:
+    log.warning('NSIDE is a required input.')
+    raise ValueError
+
 # Divide the DESI footprint into healpix pixels.
 if args.nside < 2:
     log.fatal('NSIDE = {} must be >=2'.format(args.nside))
@@ -51,7 +55,6 @@ if args.nside < 2:
 if args.tiles and args.healpixels:
     log.error('use --tiles or --healpixels but not both')
     sys.exit(1)
-
 
 if args.tiles is not None:
     if args.tiles.endswith('.ecsv'):
@@ -64,23 +67,18 @@ if args.tiles is not None:
 else:
     healpixels = np.array(args.healpixels)
 
-keep = list()
-for i, pixnum in enumerate(healpixels):
-    truthfile = mockio.findfile('truth', args.nside, pixnum, basedir=args.output_dir)
-    if not os.path.exists(truthfile):
-        keep.append(i)
-
-log.info('{}/{} pixels remaining to do'.format(len(keep), len(healpixels)))
-healpixels = healpixels[keep]
-
-# Optionally read the "real" target catalog.
-if args.realtargets is not None:
-    from astropy.io import fits
-    log.info('Loading real targets from {}'.format(args.realtargets))
-    realtargets = fits.getdata(args.realtargets, 1)
+if args.overwrite:
+    log.info('Processing {} pixel(s).'.format(len(healpixels)))
 else:
-    realtargets = None
-
+    keep = list()
+    for i, pixnum in enumerate(healpixels):
+        truthfile = mockio.findfile('truth', args.nside, pixnum, basedir=args.output_dir)
+        if not os.path.exists(truthfile):
+            keep.append(i)
+
+    log.info('{}/{} pixels remaining to do'.format(len(keep), len(healpixels)))
+    healpixels = healpixels[keep]
+
 if args.join:
     from desitarget.mock.build import join_targets_truth
     join_targets_truth(args.output_dir, nside=args.nside, healpixels=healpixels, verbose=args.verbose)
@@ -103,13 +101,13 @@ with open(args.config, 'r') as pfile:
 
 log.info('Calling targets_truth at {}'.format(time.asctime()))
 if args.no_spectra:
-    targets_truth_no_spectra(params, output_dir=args.output_dir,seed=args.seed,
+    from desitarget.mock.build import targets_truth_no_spectra
+    targets_truth_no_spectra(params, output_dir=args.output_dir, seed=args.seed,
                   verbose=args.verbose, nproc=args.nproc, nside=args.nside,
                   healpixels=healpixels)
 else:
-    targets_truth(params, args.output_dir, realtargets=realtargets, seed=args.seed,
-                  verbose=args.verbose, nproc=args.nproc, nside=args.nside,
-                  healpixels=healpixels)
+    targets_truth(params, output_dir=args.output_dir, seed=args.seed, nside=args.nside,
+                  verbose=args.verbose, nproc=args.nproc, healpixels=healpixels)
 
 
 log.info('All done at {}'.format(time.asctime()))

doc/api.rst

@@ -38,21 +38,15 @@ desitarget API
 .. automodule:: desitarget.mock.io
     :members:
 
-.. automodule:: desitarget.mock.QA
+.. automodule:: desitarget.mock.mockmaker
     :members:
 
 .. automodule:: desitarget.mock.sample
     :members:
 
-.. automodule:: desitarget.mock.selection
-    :members:
-
 .. automodule:: desitarget.mock.sfdmap
     :members:
 
-.. automodule:: desitarget.mock.spectra
-    :members:
-
 .. automodule:: desitarget.mtl
     :members:
 

doc/changes.rst

@@ -5,9 +5,11 @@ desitarget Change Log
 0.18.2 (unreleased)
 -------------------
 
+* Major refactor of select_mock_targets code infrastructure[`PR #264`_].
 * Remove extraneous imports from __init__.py, fix read the docs build
   [`PR #282`_].
 
+.. _`PR #264`: https://github.com/desihub/desitarget/pull/264
 .. _`PR #282`: https://github.com/desihub/desitarget/pull/282
 
 0.18.1 (2018-02-23)

doc/mock_example/input.yaml

@@ -1,104 +1,63 @@
-#mock target configuration file
-
+# mock target configuration file
 dust_dir: /project/projectdirs/desi/software/edison/dust/v0_1/maps
-
-decals_brick_info: /project/projectdirs/desi/target/catalogs/brick-info-dr3.v0.0.1.fits
-
-density_fluctuations: True
-
-subset:
-    ra_dec_cut: True
-    min_ra: 200
-    max_ra: 210
-    min_dec: -8
-    max_dec: 8
 sources:
-    SKY: {
-        target_name: SKY,
-        truth_name: SKY,
-        root_mock_dir: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.1/2048,
-        mock_name: random,
-        format: gaussianfield,
-        density: 1400.0,
-        selection: ndens
-    }
-    STD_FSTAR: {
-       target_name: STD_FSTAR,
-       truth_name: STD_FSTAR,
-       mag_bright: 16.0, 
-       grcolor : 0.32,
-       rzcolor : 0.13,
-       colortol : 0.16,
-       mag_faint: 19.0, 
-       root_mock_dir: /project/projectdirs/desi/mocks/mws/galaxia/alpha/v0.0.2/bricks,
-       format: galaxia,
-       selection: mag
+    BGS: {
+        target_name: BGS,
+        mockfile: /project/projectdirs/desi/mocks/bgs/MXXL/desi_footprint/v0.0.4/BGS.hdf5,
+        format: durham_mxxl_hdf5,
+        magcut: 20.2,
     }
     ELG: {
         target_name: ELG,
-        truth_name: ELG,
-        root_mock_dir: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.4,
+        mockfile: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.7_2LPT/ELG.fits,
         format: gaussianfield,
-        selection: ndens
+        density: 2155,
     }
     LRG: {
         target_name: LRG,
-        truth_name: LRG,
-        root_mock_dir: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.4,
+        mockfile: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.7_2LPT/LRG.fits,
         format: gaussianfield,
-        selection: ndens
+        density: 480,
     }
     QSO: {
-       target_name: QSO,
-      truth_name: QSO,
-       min_z : 0.0,
-       max_z : 2.1,
-       root_mock_dir: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.4,
-       format: gaussianfield,
-       selection : ndens,
-    }
-    BADQSO: {
-       target_name: QSO,
-       truth_name: STD_FSTAR,
-       root_mock_dir: /project/projectdirs/desi/mocks/mws/galaxia/alpha/v0.0.2/bricks,
-       format: galaxia,
-       global_density: True,
-       selection: ndens
+        target_name: QSO,
+        mockfile: /project/projectdirs/desi/mocks/GaussianRandomField/v0.0.7_2LPT/QSO.fits,
+        format: gaussianfield,
+        density: 120,
     }
     LYA: {
-       target_name: QSO,
-       truth_name: QSO,
-       root_mock_dir: /project/projectdirs/desi/mocks/lya_forest/v0.0.2,
-       format: lya,
-       selection : ndens
-    }
-    BGS: {
-       target_name: BGS,
-       truth_name: BGS,
-       mag_bright: 15.0,
-       mag_faintest: 20.0,
-       mag_priority_split: 19.5,
-       root_mock_dir: /project/projectdirs/desi/mocks/bgs/MXXL/desi_footprint/v0.0.3,
-       format: durham_mxxl_hdf5,
-       selection: mag
+        target_name: LYA,
+        mockfile: /project/projectdirs/desi/mocks/lya_forest/v2.0.2/master.fits,
+        format: CoLoRe,
+        nside_lya: 16,
+        density: 50,
     }
     MWS_MAIN: {
-       target_name: MWS_MAIN,
-       truth_name: MWS_MAIN,
-       mag_bright: 15.0, 
-       mag_faint_filler: 19.0, 
-       mag_faintest: 20.0, 
-       # root_mock_dir: /project/projectdirs/desi/users/forero/galaxia_small,
-       root_mock_dir: /project/projectdirs/desi/mocks/mws/galaxia/alpha/v0.0.2/bricks,
-       format: galaxia,
-       selection: mag
+        target_name: MWS_MAIN,
+        mockfile: /project/projectdirs/desi/mocks/mws/galaxia/alpha/v0.0.5/healpix,
+        nside_galaxia: 8,
+        format: galaxia,
+    }
+    MWS_NEARBY: {
+        target_name: MWS_NEARBY,
+        mockfile: /project/projectdirs/desi/mocks/mws/100pc/v0.0.3/mock_100pc.fits,
+        format: mws_100pc,
+    }
+    WD: {
+        target_name: WD,
+        mockfile: /project/projectdirs/desi/mocks/mws/wd/v0.0.2/mock_wd.fits,
+        format: mws_wd,
+    }
+    FAINTSTAR: {
+        target_name: FAINTSTAR,
+        mockfile: /project/projectdirs/desi/mocks/mws/galaxia/alpha/0.0.5_superfaint/healpix,
+        nside_galaxia: 8,
+        format: galaxia,
+        magcut: 23.5,
     }
-    MWS_WD: {
-        target_name: MWS_WD,
-        truth_name: MWS_WD,
-        mag_bright: 15.0, 
-        mag_faint: 20.0, 
-        root_mock_dir: /project/projectdirs/desi/mocks/mws/wd100pc/v0.0.1,
-        format: wd100pc,
-        selection: mag
+    SKY: {
+        target_name: SKY,
+        mockfile: /project/projectdirs/desi/mocks/uniformsky/0.1/uniformsky-2048-0.1.fits,
+        format: uniformsky,
+        density: 1400,
     }