'
)
},
@@ -151,14 +151,14 @@ $(document).ready(function() {
columns: dataConf.colsFields,
order: dataConf.order,
searchDelay: 2000,
- dom : dom,
+ dom: dom,
buttons: DEFAULT_DATATABLE_BUTTONS
};
// apply deferLoading config, if supplied
if (dataConf.hasOwnProperty('deferLoading')) {
dataTableConf.deferLoading = dataConf.deferLoading;
// change the message printed in the empty table if deferLoading active
- dataTableConf.initComplete = function(settings, json) {
+ dataTableConf.initComplete = function (settings, json) {
$("td.dataTables_empty").text("Submit a query to view results");
}
}
@@ -171,9 +171,9 @@ $(document).ready(function() {
// ...
// ];
let dataSet = [];
- dataConf.dataQuery.forEach( function(obj) {
+ dataConf.dataQuery.forEach(function (obj) {
let row = [];
- dataConf.colsFields.forEach(function(elem) {
+ dataConf.colsFields.forEach(function (elem) {
row.push(obj[elem])
})
dataSet.push(row)
@@ -201,106 +201,106 @@ $(document).ready(function() {
{
"targets": 1,
"data": "name",
- "render": function ( data, type, row, meta ) {
+ "render": function (data, type, row, meta) {
return '' + row[1] + '';
}
},
{
"targets": 3,
"data": "image",
- "render": function ( data, type, row, meta ) {
+ "render": function (data, type, row, meta) {
return '' + row[3] + '';
}
},
{
"targets": 4,
"data": "frequency",
- "render": function ( data, type, row, meta ) {
+ "render": function (data, type, row, meta) {
return row[4].toFixed(2);
}
},
{
"targets": 5,
"data": "ra",
- "render": function ( data, type, row, meta ) {
+ "render": function (data, type, row, meta) {
return row[5].toFixed(4);
}
},
{
"targets": 6,
"data": "ra_err",
- "render": function ( data, type, row, meta ) {
- return (row[6] * 3600.).toFixed(4);
+ "render": function (data, type, row, meta) {
+ return (row[6] * 3600.).toFixed(4);
}
},
{
"targets": 7,
"data": "dec",
- "render": function ( data, type, row, meta ) {
- return row[7].toFixed(4);
+ "render": function (data, type, row, meta) {
+ return row[7].toFixed(4);
}
},
{
"targets": 8,
"data": "dec_err",
- "render": function ( data, type, row, meta ) {
- return (row[8] * 3600.).toFixed(4);
+ "render": function (data, type, row, meta) {
+ return (row[8] * 3600.).toFixed(4);
}
},
{
"targets": 9,
"data": "flux_peak",
- "render": function ( data, type, row, meta ) {
- return (row[9]).toFixed(3);
+ "render": function (data, type, row, meta) {
+ return (row[9]).toFixed(3);
}
},
{
"targets": 10,
"data": "flux_peak_err",
- "render": function ( data, type, row, meta ) {
- return (row[10]).toFixed(3);
+ "render": function (data, type, row, meta) {
+ return (row[10]).toFixed(3);
}
},
{
"targets": 11,
"data": "flux_peak_isl_ratio",
- "render": function ( data, type, row, meta ) {
- return (row[11]).toFixed(2);
+ "render": function (data, type, row, meta) {
+ return (row[11]).toFixed(2);
}
},
{
"targets": 12,
"data": "flux_int",
- "render": function ( data, type, row, meta ) {
- return (row[12]).toFixed(3);
+ "render": function (data, type, row, meta) {
+ return (row[12]).toFixed(3);
}
},
{
"targets": 13,
"data": "flux_int_err",
- "render": function ( data, type, row, meta ) {
- return (row[13]).toFixed(3);
+ "render": function (data, type, row, meta) {
+ return (row[13]).toFixed(3);
}
},
{
"targets": 14,
"data": "flux_int_isl_ratio",
- "render": function ( data, type, row, meta ) {
- return (row[14]).toFixed(2);
+ "render": function (data, type, row, meta) {
+ return (row[14]).toFixed(2);
}
},
{
"targets": 15,
"data": "local_rms",
- "render": function ( data, type, row, meta ) {
- return (row[15]).toFixed(2);
+ "render": function (data, type, row, meta) {
+ return (row[15]).toFixed(2);
}
},
{
"targets": 16,
"data": "snr",
- "render": function ( data, type, row, meta ) {
- return (row[16]).toFixed(2);
+ "render": function (data, type, row, meta) {
+ return (row[16]).toFixed(2);
}
},
{
@@ -318,7 +318,7 @@ $(document).ready(function() {
drawExternalResultsTable('#externalResultsTable');
// Trigger the update search on the datatable
- $("#catalogSearch").on('click', function(e) {
+ $("#catalogSearch").on('click', function (e) {
let PipeRun = document.getElementById("runSelect");
let qry_url = dataConfParsed.api;
if (PipeRun.value != '') {
@@ -338,7 +338,7 @@ $(document).ready(function() {
qry_url = qry_url + "&coord=" + encodeURIComponent(coord.value);
};
if (unit.value) {
- qry_url = qry_url + "&radiusunit=" + unit.value
+ qry_url = qry_url + "&radiusunit=" + unit.value
}
let avg_flux_type = document.getElementById("aveFluxSelect");
let avg_flux_min = document.getElementById("avgFluxMinSelect");
@@ -510,7 +510,7 @@ $(document).ready(function() {
qry_url = qry_url + "&source_selection=" + source_selection.value;
};
if (source_selection_type.value) {
- qry_url = qry_url + "&source_selection_type=" +source_selection_type.value;
+ qry_url = qry_url + "&source_selection_type=" + source_selection_type.value;
};
table.ajax.url(qry_url);
table.ajax.reload();
@@ -522,8 +522,8 @@ $(document).ready(function() {
});
// Trigger the search reset on the datatable
- $("#resetSearch").on('click', function(e) {
- $('#runSelect option').prop('selected', function() {
+ $("#resetSearch").on('click', function (e) {
+ $('#runSelect option').prop('selected', function () {
return this.defaultSelected
});
let inputs = [
@@ -536,7 +536,7 @@ $(document).ready(function() {
'compactnessMinSelect', 'compactnessMaxSelect', 'objectNameInput', 'MinSnrMinSelect',
'MinSnrMaxSelect', 'MaxSnrMinSelect', 'MaxSnrMaxSelect', 'fluxMaxMinSelect',
'fluxMaxMaxSelect', 'sourceSelectionSelect',
- ];
+ ];
var input;
for (input of inputs) {
document.getElementById(input).value = '';
diff --git a/templates/sources_etav_plot.html b/templates/sources_etav_plot.html
index 6432efe87..8b16f79d1 100644
--- a/templates/sources_etav_plot.html
+++ b/templates/sources_etav_plot.html
@@ -149,7 +149,7 @@
Selected Source Light C
function update_card(id) {
$('#cardsUpdate').html('').load(
- "{% url 'vast_pipeline:source_etav_plot_update' 0 %}".replace('0', id),
+ "{% url 'vast_pipeline:source_etav_plot_update' 'e1f6bf88-4b07-4f28-bf9b-3ccf9d2ca813' %}".replace('e1f6bf88-4b07-4f28-bf9b-3ccf9d2ca813', id),
function() {
// redraw the external results table, function is defined in datatables-pipeline.js
drawExternalResultsTable('#externalResultsTable');
diff --git a/vast_pipeline/image/main.py b/vast_pipeline/image/main.py
index 2a8edaf5b..98d5d0d48 100644
--- a/vast_pipeline/image/main.py
+++ b/vast_pipeline/image/main.py
@@ -4,6 +4,7 @@
import os
import logging
+import uuid
import numpy as np
import pandas as pd
@@ -124,34 +125,31 @@ def __get_header(self, hdu_index: int) -> fits.Header:
hdu = hdulist[hdu_index]
except Exception:
raise IOError((
- 'Could not read FITS file: '
- f'{self.path}'
+ f"Could not read FITS file: {self.path}"
))
return hdu.header.copy()
def __set_img_attr_for_telescope(self, header):
- '''
+ """
Set the image attributes depending on the telescope type
- '''
- self.polarisation = header.get('STOKES', 'I')
- self.duration = float(header.get('DURATION', 0.))
- self.beam_bmaj = 0.
- self.beam_bmin = 0.
- self.beam_bpa = 0.
+ """
+ self.polarisation = header.get("STOKES", "I")
+ self.duration = float(header.get("DURATION", 0.0))
+ self.beam_bmaj = 0.0
+ self.beam_bmin = 0.0
+ self.beam_bpa = 0.0
self.ra = None
self.dec = None
self.fov_bmaj = None
self.fov_bmin = None
- if header.get('TELESCOP', None) == 'ASKAP':
+ if header.get("TELESCOP", None) == "ASKAP":
try:
- self.datetime = pd.Timestamp(
- header['DATE-OBS'], tz=header['TIMESYS']
- )
- self.beam_bmaj = header['BMAJ']
- self.beam_bmin = header['BMIN']
- self.beam_bpa = header['BPA']
+ self.datetime = pd.Timestamp(header["DATE-OBS"], tz=header["TIMESYS"])
+ self.beam_bmaj = header["BMAJ"]
+ self.beam_bmin = header["BMIN"]
+ self.beam_bpa = header["BPA"]
except KeyError as e:
logger.exception(
"Image %s does not contain expected FITS header keywords.",
@@ -160,9 +158,9 @@ def __set_img_attr_for_telescope(self, header):
raise e
params = {
- 'header': header,
- 'fits_naxis1': 'NAXIS1',
- 'fits_naxis2': 'NAXIS2',
+ "header": header,
+ "fits_naxis1": "NAXIS1",
+ "fits_naxis2": "NAXIS2",
}
# set the coordinate attributes
@@ -187,17 +185,17 @@ def __get_img_coordinates(
None
"""
wcs = WCS(header, naxis=2)
- pix_centre = [[header[fits_naxis1] / 2., header[fits_naxis2] / 2.]]
+ pix_centre = [[header[fits_naxis1] / 2.0, header[fits_naxis2] / 2.0]]
self.ra, self.dec = wcs.wcs_pix2world(pix_centre, 1)[0]
# The field-of-view (in pixels) is assumed to be a circle in the centre
# of the image. This may be an ellipse on the sky, eg MOST images.
# We leave a pixel margin at the edge that we don't use.
# TODO: move unused pixel as argument
- unusedpix = 0.
- usable_radius_pix = self.__get_radius_pixels(
- header, fits_naxis1, fits_naxis2
- ) - unusedpix
+ unusedpix = 0.0
+ usable_radius_pix = (
+ self.__get_radius_pixels(header, fits_naxis1, fits_naxis2) - unusedpix
+ )
cdelt1, cdelt2 = proj_plane_pixel_scales(WCS(header).celestial)
self.fov_bmin = usable_radius_pix * abs(cdelt1)
self.fov_bmaj = usable_radius_pix * abs(cdelt2)
@@ -232,7 +230,7 @@ def __get_radius_pixels(
else:
# We simply place the largest circle we can in the centre.
diameter = min(header[fits_naxis1], header[fits_naxis2])
- return diameter / 2.
+ return diameter / 2.0
def __get_frequency(self, header: fits.Header) -> None:
"""
@@ -248,16 +246,16 @@ def __get_frequency(self, header: fits.Header) -> None:
self.freq_eff = None
self.freq_bw = None
try:
- freq_keys = ('FREQ', 'VOPT')
- if ('ctype3' in header) and (header['ctype3'] in freq_keys):
- self.freq_eff = header['crval3']
- self.freq_bw = header['cdelt3'] if 'cdelt3' in header else 0.0
- elif ('ctype4' in header) and (header['ctype4'] in freq_keys):
- self.freq_eff = header['crval4']
- self.freq_bw = header['cdelt4'] if 'cdelt4' in header else 0.0
+ freq_keys = ("FREQ", "VOPT")
+ if ("ctype3" in header) and (header["ctype3"] in freq_keys):
+ self.freq_eff = header["crval3"]
+ self.freq_bw = header["cdelt3"] if "cdelt3" in header else 0.0
+ elif ("ctype4" in header) and (header["ctype4"] in freq_keys):
+ self.freq_eff = header["crval4"]
+ self.freq_bw = header["cdelt4"] if "cdelt4" in header else 0.0
else:
- self.freq_eff = header['restfreq']
- self.freq_bw = header['restbw'] if 'restbw' in header else 0.0
+ self.freq_eff = header["restfreq"]
+ self.freq_bw = header["restbw"] if "restbw" in header else 0.0
except Exception:
msg = f"Frequency not specified in headers for {self.name}"
logger.error(msg)
@@ -300,9 +298,9 @@ def __init__(
None.
"""
# inherit from parent
- self.selavy_path = paths['selavy'][path]
- self.noise_path = paths['noise'].get(path, '')
- self.background_path = paths['background'].get(path, '')
+ self.selavy_path = paths["selavy"][path]
+ self.noise_path = paths["noise"].get(path, "")
+ self.background_path = paths["background"].get(path, "")
self.config: Dict = config
super().__init__(path, hdu_index)
@@ -340,20 +338,23 @@ def read_selavy(self, dj_image: models.Image) -> pd.DataFrame:
# fix dtype of columns
for ky in tr_selavy:
key = tr_selavy[ky]
- if df[key['name']].dtype != key['dtype']:
- df[key['name']] = df[key['name']].astype(key['dtype'])
+ if df[key["name"]].dtype != key["dtype"]:
+ df[key["name"]] = df[key["name"]].astype(key["dtype"])
+
+ # Add id column
+ df["id"] = df.apply(lambda _: str(uuid.uuid4()), axis=1)
# do checks and fill in missing field for uploading sources
# in DB (see fields in models.py -> Source model)
- if df['component_id'].duplicated().any():
- raise Exception('Found duplicated names in sources')
+ if df["component_id"].duplicated().any():
+ raise Exception("Found duplicated names in sources")
# drop unrealistic sources
cols_to_check = [
- 'bmaj',
- 'bmin',
- 'flux_peak',
- 'flux_int',
+ "bmaj",
+ "bmin",
+ "flux_peak",
+ "flux_int",
]
bad_sources = df[(df[cols_to_check] == 0).any(axis=1)]
@@ -364,117 +365,117 @@ def read_selavy(self, dj_image: models.Image) -> pd.DataFrame:
# dropping tiny sources
nr_sources_old = df.shape[0]
df = df.loc[
- (df['bmaj'] > dj_image.beam_bmaj * 500) &
- (df['bmin'] > dj_image.beam_bmin * 500)
+ (df["bmaj"] > dj_image.beam_bmaj * 500)
+ & (df["bmin"] > dj_image.beam_bmin * 500)
]
if df.shape[0] != nr_sources_old:
- logger.info(
- 'Dropped %i tiny sources.', nr_sources_old - df.shape[0]
- )
+ logger.info("Dropped %i tiny sources.", nr_sources_old - df.shape[0])
# add fields from image and fix name column
- df['image_id'] = dj_image.id
- df['time'] = dj_image.datetime
+ df["image_id"] = str(dj_image.id)
+ df["time"] = dj_image.datetime
# append img prefix to source name
- img_prefix = dj_image.name.split('.i.', 1)[-1].split('.', 1)[0] + '_'
- df['name'] = img_prefix + df['component_id']
+ img_prefix = dj_image.name.split(".i.", 1)[-1].split(".", 1)[0] + "_"
+ df["name"] = img_prefix + df["component_id"]
# # fix error fluxes
- for col in ['flux_int_err', 'flux_peak_err']:
+ for col in ["flux_int_err", "flux_peak_err"]:
sel = df[col] < settings.FLUX_DEFAULT_MIN_ERROR
if sel.any():
df.loc[sel, col] = settings.FLUX_DEFAULT_MIN_ERROR
# # fix error ra dec
- for col in ['ra_err', 'dec_err']:
+ for col in ["ra_err", "dec_err"]:
sel = df[col] < settings.POS_DEFAULT_MIN_ERROR
if sel.any():
df.loc[sel, col] = settings.POS_DEFAULT_MIN_ERROR
- df[col] = df[col] / 3600.
+ df[col] = df[col] / 3600.0
# replace 0 local_rms values using user config value
- df.loc[
- df['local_rms'] == 0., 'local_rms'
- ] = self.config["selavy_local_rms_fill_value"]
+ df.loc[df["local_rms"] == 0.0, "local_rms"] = self.config[
+ "selavy_local_rms_fill_value"
+ ]
- df['snr'] = df['flux_peak'].values / df['local_rms'].values
- df['compactness'] = df['flux_int'].values / df['flux_peak'].values
+ df["snr"] = df["flux_peak"].values / df["local_rms"].values
+ df["compactness"] = df["flux_int"].values / df["flux_peak"].values
if self.config["condon_errors"]:
logger.debug("Calculating Condon '97 errors...")
theta_B = dj_image.beam_bmaj
theta_b = dj_image.beam_bmin
- df[[
- 'flux_peak_err',
- 'flux_int_err',
- 'err_bmaj',
- 'err_bmin',
- 'err_pa',
- 'ra_err',
- 'dec_err',
- ]] = df[[
- 'flux_peak',
- 'flux_int',
- 'bmaj',
- 'bmin',
- 'pa',
- 'snr',
- 'local_rms',
- ]].apply(
+ df[
+ [
+ "flux_peak_err",
+ "flux_int_err",
+ "err_bmaj",
+ "err_bmin",
+ "err_pa",
+ "ra_err",
+ "dec_err",
+ ]
+ ] = df[
+ [
+ "flux_peak",
+ "flux_int",
+ "bmaj",
+ "bmin",
+ "pa",
+ "snr",
+ "local_rms",
+ ]
+ ].apply(
calc_condon_flux_errors,
args=(theta_B, theta_b),
axis=1,
- result_type='expand'
+ result_type="expand",
)
logger.debug("Condon errors done.")
logger.debug("Calculating positional errors...")
# TODO: avoid extra column given that it is a single value
- df['ew_sys_err'] = self.config["ra_uncertainty"] / 3600.
- df['ns_sys_err'] = self.config["dec_uncertainty"] / 3600.
-
- df['error_radius'] = calc_error_radius(
- df['ra'].values,
- df['ra_err'].values,
- df['dec'].values,
- df['dec_err'].values,
+ df["ew_sys_err"] = self.config["ra_uncertainty"] / 3600.0
+ df["ns_sys_err"] = self.config["dec_uncertainty"] / 3600.0
+
+ df["error_radius"] = calc_error_radius(
+ df["ra"].values,
+ df["ra_err"].values,
+ df["dec"].values,
+ df["dec_err"].values,
)
- df['uncertainty_ew'] = np.hypot(
- df['ew_sys_err'].values, df['error_radius'].values
+ df["uncertainty_ew"] = np.hypot(
+ df["ew_sys_err"].values, df["error_radius"].values
)
- df['uncertainty_ns'] = np.hypot(
- df['ns_sys_err'].values, df['error_radius'].values
+ df["uncertainty_ns"] = np.hypot(
+ df["ns_sys_err"].values, df["error_radius"].values
)
# weight calculations to use later
- df['weight_ew'] = 1. / df['uncertainty_ew'].values**2
- df['weight_ns'] = 1. / df['uncertainty_ns'].values**2
+ df["weight_ew"] = 1.0 / df["uncertainty_ew"].values ** 2
+ df["weight_ns"] = 1.0 / df["uncertainty_ns"].values ** 2
- logger.debug('Positional errors done.')
+ logger.debug("Positional errors done.")
# Initialise the forced column as False
- df['forced'] = False
+ df["forced"] = False
# Calculate island flux fractions
island_flux_totals = (
- df[['island_id', 'flux_int', 'flux_peak']]
- .groupby('island_id')
- .agg('sum')
+ df[["island_id", "flux_int", "flux_peak"]].groupby("island_id").agg("sum")
)
- df['flux_int_isl_ratio'] = (
- df['flux_int'].values
- / island_flux_totals.loc[df['island_id']]['flux_int'].values
+ df["flux_int_isl_ratio"] = (
+ df["flux_int"].values
+ / island_flux_totals.loc[df["island_id"]]["flux_int"].values
)
- df['flux_peak_isl_ratio'] = (
- df['flux_peak'].values
- / island_flux_totals.loc[df['island_id']]['flux_peak'].values
+ df["flux_peak_isl_ratio"] = (
+ df["flux_peak"].values
+ / island_flux_totals.loc[df["island_id"]]["flux_peak"].values
)
return df
diff --git a/vast_pipeline/migrations/0001_initial.py b/vast_pipeline/migrations/0001_initial.py
index 6b8377bdc..5d710c2a6 100644
--- a/vast_pipeline/migrations/0001_initial.py
+++ b/vast_pipeline/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.7 on 2021-01-18 05:54
+# Generated by Django 3.2.19 on 2023-07-10 11:05
from django.conf import settings
import django.core.validators
@@ -6,6 +6,7 @@
import django.db.models.deletion
import tagulous.models.fields
import tagulous.models.models
+import uuid
class Migration(migrations.Migration):
@@ -13,15 +14,15 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
- ('contenttypes', '0002_remove_content_type_name'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('contenttypes', '0002_remove_content_type_name'),
]
operations = [
migrations.CreateModel(
name='Association',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('d2d', models.FloatField(default=0.0, help_text='astronomical distance calculated by Astropy, arcsec.')),
('dr', models.FloatField(default=0.0, help_text='De Ruiter radius calculated in advanced association.')),
],
@@ -29,7 +30,7 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='Band',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(max_length=12, unique=True)),
('frequency', models.FloatField(help_text='central frequency of band (integer MHz)')),
('bandwidth', models.FloatField(help_text='bandwidth (MHz)')),
@@ -38,20 +39,10 @@ class Migration(migrations.Migration):
'ordering': ['frequency'],
},
),
- migrations.CreateModel(
- name='CrossMatch',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('manual', models.BooleanField()),
- ('distance', models.FloatField()),
- ('probability', models.FloatField()),
- ('comment', models.TextField(blank=True, default='', max_length=1000)),
- ],
- ),
migrations.CreateModel(
name='Image',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('measurements_path', models.FilePathField(db_column='meas_path', help_text='the path to the measurements parquet that belongs to this image', max_length=200)),
('polarisation', models.CharField(choices=[('I', 'I'), ('XX', 'XX'), ('YY', 'YY'), ('Q', 'Q'), ('U', 'U'), ('V', 'V')], help_text='Polarisation of the image one of I,XX,YY,Q,U,V.', max_length=2)),
('name', models.CharField(help_text='Name of the image.', max_length=200, unique=True)),
@@ -74,74 +65,32 @@ class Migration(migrations.Migration):
('rms_median', models.FloatField(help_text='Background average RMS from the provided RMS map (mJy).')),
('rms_min', models.FloatField(help_text='Background minimum RMS from the provided RMS map (mJy).')),
('rms_max', models.FloatField(help_text='Background maximum RMS from the provided RMS map (mJy).')),
- ('band', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Band')),
+ ('band', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.band')),
],
options={
'ordering': ['datetime'],
},
),
- migrations.CreateModel(
- name='Measurement',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('name', models.CharField(max_length=64, unique=True)),
- ('ra', models.FloatField(help_text='RA of the source (Deg).')),
- ('ra_err', models.FloatField(help_text='RA error of the source (Deg).')),
- ('dec', models.FloatField(help_text='DEC of the source (Deg).')),
- ('dec_err', models.FloatField(help_text='DEC error of the source (Deg).')),
- ('bmaj', models.FloatField(help_text='The major axis of the Gaussian fit to the source (Deg).')),
- ('err_bmaj', models.FloatField(help_text='Error major axis (Deg).')),
- ('bmin', models.FloatField(help_text='The minor axis of the Gaussian fit to the source (Deg).')),
- ('err_bmin', models.FloatField(help_text='Error minor axis (Deg).')),
- ('pa', models.FloatField(help_text='Position angle of Gaussian fit east of north to bmaj (Deg).')),
- ('err_pa', models.FloatField(help_text='Error position angle (Deg).')),
- ('ew_sys_err', models.FloatField(help_text='Systematic error in east-west (RA) direction (Deg).')),
- ('ns_sys_err', models.FloatField(help_text='Systematic error in north-south (dec) direction (Deg).')),
- ('error_radius', models.FloatField(help_text='Estimate of maximum error radius using ra_err and dec_err (Deg).')),
- ('uncertainty_ew', models.FloatField(help_text='Total east-west (RA) uncertainty, quadratic sum of error_radius and ew_sys_err (Deg).')),
- ('uncertainty_ns', models.FloatField(help_text='Total north-south (Dec) uncertainty, quadratic sum of error_radius and ns_sys_err (Deg).')),
- ('flux_int', models.FloatField()),
- ('flux_int_err', models.FloatField()),
- ('flux_int_isl_ratio', models.FloatField(help_text='Ratio of the component integrated flux to the total island integrated flux.')),
- ('flux_peak', models.FloatField()),
- ('flux_peak_err', models.FloatField()),
- ('flux_peak_isl_ratio', models.FloatField(help_text='Ratio of the component peak flux to the total island peak flux.')),
- ('chi_squared_fit', models.FloatField(db_column='chi2_fit', help_text='Chi-squared of the Guassian fit to the source.')),
- ('spectral_index', models.FloatField(db_column='spectr_idx', help_text='In-band Selavy spectral index.')),
- ('spectral_index_from_TT', models.BooleanField(db_column='spectr_idx_tt', default=False, help_text='True/False if the spectral index came from the taylor term.')),
- ('local_rms', models.FloatField(help_text='Local rms in mJy from Selavy.')),
- ('snr', models.FloatField(help_text='Signal-to-noise ratio of the measurement.')),
- ('flag_c4', models.BooleanField(default=False, help_text='Fit flag from Selavy.')),
- ('compactness', models.FloatField(help_text='Int flux over peak flux.')),
- ('has_siblings', models.BooleanField(default=False, help_text='True if the fit comes from an island that has more than 1 component.')),
- ('component_id', models.CharField(help_text='The ID of the component from which the source comes from.', max_length=64)),
- ('island_id', models.CharField(help_text='The ID of the island from which the source comes from.', max_length=64)),
- ('forced', models.BooleanField(default=False, help_text='True: the measurement is forced extracted.')),
- ('image', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Image')),
- ],
- options={
- 'ordering': ['ra'],
- },
- ),
migrations.CreateModel(
name='RelatedSource',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
],
),
migrations.CreateModel(
name='Run',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(help_text='name of the pipeline run', max_length=64, unique=True, validators=[django.core.validators.RegexValidator(inverse_match=True, message='Name contains not allowed characters!', regex='[\\[@!#$%^&*()<>?/\\|}{~:\\] ]')])),
('description', models.CharField(blank=True, help_text='A short description of the pipeline run.', max_length=240)),
('time', models.DateTimeField(auto_now=True, help_text='Datetime of a pipeline run.')),
('path', models.FilePathField(help_text='path to the pipeline run', max_length=200)),
- ('status', models.CharField(choices=[('INI', 'Initialised'), ('QUE', 'Queued'), ('RUN', 'Running'), ('END', 'Completed'), ('ERR', 'Error'), ('RES', 'Restoring')], default='INI', help_text='Status of the pipeline run.', max_length=3)),
+ ('status', models.CharField(choices=[('INI', 'Initialised'), ('QUE', 'Queued'), ('RUN', 'Running'), ('END', 'Completed'), ('ERR', 'Error'), ('RES', 'Restoring'), ('DEL', 'Deleting')], default='INI', help_text='Status of the pipeline run.', max_length=3)),
('n_images', models.IntegerField(default=0, help_text='number of images processed in this run')),
('n_sources', models.IntegerField(default=0, help_text='number of sources extracted in this run')),
('n_selavy_measurements', models.IntegerField(default=0, help_text='number of selavy measurements in this run')),
('n_forced_measurements', models.IntegerField(default=0, help_text='number of forced measurements in this run')),
+ ('n_new_sources', models.IntegerField(default=0, help_text='number of new sources in this run')),
('epoch_based', models.BooleanField(default=False, help_text='Whether the run was processed using epoch based association, i.e. the user passed in groups of images defining epochs rather than every image being treated individually.')),
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
],
@@ -152,7 +101,7 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='Source',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(max_length=100)),
('new', models.BooleanField(default=False, help_text='New Source.')),
('wavg_ra', models.FloatField(help_text='The weighted average right ascension (Deg).')),
@@ -176,32 +125,22 @@ class Migration(migrations.Migration):
('eta_peak', models.FloatField(help_text='Eta metric for peak flux.')),
('new_high_sigma', models.FloatField(help_text='The largest sigma value for the new source if it was placed in previous image.')),
('n_neighbour_dist', models.FloatField(help_text='Distance to the nearest neighbour (deg)')),
- ('vs_abs_significant_max_int', models.FloatField(default=0.0, help_text='Maximum value of all measurement pair variability t-statistics for int flux that exceed SOURCE_AGGREGATE_PAIR_METRICS_MIN_ABS_VS in the pipeline run configuration.')),
- ('m_abs_significant_max_int', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for int flux that exceed SOURCE_AGGREGATE_PAIR_METRICS_MIN_ABS_VS in the pipeline run configuration.')),
- ('vs_abs_significant_max_peak', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair variability t-statistics for peak flux that exceed SOURCE_AGGREGATE_PAIR_METRICS_MIN_ABS_VS in the pipeline run configuration.')),
- ('m_abs_significant_max_peak', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for peak flux that exceed SOURCE_AGGREGATE_PAIR_METRICS_MIN_ABS_VS in the pipeline run configuration.')),
+ ('vs_abs_significant_max_int', models.FloatField(default=0.0, help_text='Maximum value of all measurement pair variability t-statistics for int flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.')),
+ ('m_abs_significant_max_int', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for int flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.')),
+ ('vs_abs_significant_max_peak', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair variability t-statistics for peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.')),
+ ('m_abs_significant_max_peak', models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.')),
('n_meas', models.IntegerField(help_text='total measurements of the source')),
('n_meas_sel', models.IntegerField(help_text='total selavy extracted measurements of the source')),
('n_meas_forced', models.IntegerField(help_text='total force extracted measurements of the source')),
('n_rel', models.IntegerField(help_text='total relations of the source with other sources')),
('n_sibl', models.IntegerField(help_text='total siblings of the source')),
+ ('related', models.ManyToManyField(through='vast_pipeline.RelatedSource', to='vast_pipeline.Source')),
+ ('run', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.run')),
],
options={
'abstract': False,
},
),
- migrations.CreateModel(
- name='Survey',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('name', models.CharField(help_text='Name of the Survey e.g. NVSS.', max_length=32, unique=True)),
- ('comment', models.TextField(blank=True, default='', max_length=1000)),
- ('frequency', models.IntegerField(help_text='Frequency of the survey.')),
- ],
- options={
- 'ordering': ['name'],
- },
- ),
migrations.CreateModel(
name='Tagulous_Source_tags',
fields=[
@@ -218,51 +157,15 @@ class Migration(migrations.Migration):
},
bases=(tagulous.models.models.BaseTagModel, models.Model),
),
- migrations.CreateModel(
- name='SurveySource',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('name', models.CharField(help_text='Name of the survey source.', max_length=100)),
- ('ra', models.FloatField(help_text='RA of the survey source (Deg).')),
- ('ra_err', models.FloatField(help_text='RA error of the survey source (Deg).')),
- ('dec', models.FloatField(help_text='DEC of the survey source (Deg).')),
- ('dec_err', models.FloatField(help_text='DEC error of the survey source (Deg).')),
- ('bmaj', models.FloatField(help_text='The major axis of the Gaussian fit to the survey source (arcsecs).')),
- ('bmin', models.FloatField(help_text='The minor axis of the Gaussian fit to the survey source (arcsecs).')),
- ('pa', models.FloatField(help_text='Position angle of Gaussian fit east of north to bmaj (Deg).')),
- ('flux_peak', models.FloatField(help_text='Peak flux of the Guassian fit (Jy).')),
- ('flux_peak_err', models.FloatField(help_text='Peak flux error of the Gaussian fit (Jy).')),
- ('flux_int', models.FloatField(help_text='Integrated flux of the Guassian fit (Jy).')),
- ('flux_int_err', models.FloatField(help_text='Integrated flux of the Guassian fit (Jy).')),
- ('alpha', models.FloatField(default=0, help_text='Spectral index of the survey source.')),
- ('image_name', models.CharField(blank=True, help_text='Name of survey image where measurement was made.', max_length=100)),
- ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Survey')),
- ],
- ),
migrations.CreateModel(
name='SourceFav',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('comment', models.TextField(blank=True, default='', help_text='Why did you include this as favourite', max_length=500)),
- ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Source')),
+ ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.source')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
),
- migrations.AddField(
- model_name='source',
- name='cross_match_sources',
- field=models.ManyToManyField(through='vast_pipeline.CrossMatch', to='vast_pipeline.SurveySource'),
- ),
- migrations.AddField(
- model_name='source',
- name='related',
- field=models.ManyToManyField(through='vast_pipeline.RelatedSource', to='vast_pipeline.Source'),
- ),
- migrations.AddField(
- model_name='source',
- name='run',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Run'),
- ),
migrations.AddField(
model_name='source',
name='tags',
@@ -271,7 +174,7 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='SkyRegion',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('centre_ra', models.FloatField()),
('centre_dec', models.FloatField()),
('width_ra', models.FloatField()),
@@ -286,30 +189,56 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='relatedsource',
name='from_source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Source'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.source'),
),
migrations.AddField(
model_name='relatedsource',
name='to_source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_sources', to='vast_pipeline.Source'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_sources', to='vast_pipeline.source'),
),
migrations.CreateModel(
- name='MeasurementPair',
+ name='Measurement',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('vs_peak', models.FloatField(help_text='Variability metric: t-statistic for peak fluxes.')),
- ('m_peak', models.FloatField(help_text='Variability metric: modulation index for peak fluxes.')),
- ('vs_int', models.FloatField(help_text='Variability metric: t-statistic for integrated fluxes.')),
- ('m_int', models.FloatField(help_text='Variability metric: modulation index for integrated fluxes.')),
- ('measurement_a', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='measurement_pairs_a', to='vast_pipeline.Measurement')),
- ('measurement_b', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='measurement_pairs_b', to='vast_pipeline.Measurement')),
- ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Source')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=64)),
+ ('ra', models.FloatField(help_text='RA of the source (Deg).')),
+ ('ra_err', models.FloatField(help_text='RA error of the source (Deg).')),
+ ('dec', models.FloatField(help_text='DEC of the source (Deg).')),
+ ('dec_err', models.FloatField(help_text='DEC error of the source (Deg).')),
+ ('bmaj', models.FloatField(help_text='The major axis of the Gaussian fit to the source (Deg).')),
+ ('err_bmaj', models.FloatField(help_text='Error major axis (Deg).')),
+ ('bmin', models.FloatField(help_text='The minor axis of the Gaussian fit to the source (Deg).')),
+ ('err_bmin', models.FloatField(help_text='Error minor axis (Deg).')),
+ ('pa', models.FloatField(help_text='Position angle of Gaussian fit east of north to bmaj (Deg).')),
+ ('err_pa', models.FloatField(help_text='Error position angle (Deg).')),
+ ('ew_sys_err', models.FloatField(help_text='Systematic error in east-west (RA) direction (Deg).')),
+ ('ns_sys_err', models.FloatField(help_text='Systematic error in north-south (dec) direction (Deg).')),
+ ('error_radius', models.FloatField(help_text='Estimate of maximum error radius using ra_err and dec_err (Deg).')),
+ ('uncertainty_ew', models.FloatField(help_text='Total east-west (RA) uncertainty, quadratic sum of error_radius and ew_sys_err (Deg).')),
+ ('uncertainty_ns', models.FloatField(help_text='Total north-south (Dec) uncertainty, quadratic sum of error_radius and ns_sys_err (Deg).')),
+ ('flux_int', models.FloatField()),
+ ('flux_int_err', models.FloatField()),
+ ('flux_int_isl_ratio', models.FloatField(help_text='Ratio of the component integrated flux to the total island integrated flux.')),
+ ('flux_peak', models.FloatField()),
+ ('flux_peak_err', models.FloatField()),
+ ('flux_peak_isl_ratio', models.FloatField(help_text='Ratio of the component peak flux to the total island peak flux.')),
+ ('chi_squared_fit', models.FloatField(db_column='chi2_fit', help_text='Chi-squared of the Guassian fit to the source.')),
+ ('spectral_index', models.FloatField(db_column='spectr_idx', help_text='In-band Selavy spectral index.')),
+ ('spectral_index_from_TT', models.BooleanField(db_column='spectr_idx_tt', default=False, help_text='True/False if the spectral index came from the taylor term.')),
+ ('local_rms', models.FloatField(help_text='Local rms in mJy from Selavy.')),
+ ('snr', models.FloatField(help_text='Signal-to-noise ratio of the measurement.')),
+ ('flag_c4', models.BooleanField(default=False, help_text='Fit flag from Selavy.')),
+ ('compactness', models.FloatField(help_text='Int flux over peak flux.')),
+ ('has_siblings', models.BooleanField(default=False, help_text='True if the fit comes from an island that has more than 1 component.')),
+ ('component_id', models.CharField(help_text='The ID of the component from which the source comes from.', max_length=64)),
+ ('island_id', models.CharField(help_text='The ID of the island from which the source comes from.', max_length=64)),
+ ('forced', models.BooleanField(default=False, help_text='True: the measurement is forced extracted.')),
+ ('image', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.image')),
+ ('source', models.ManyToManyField(through='vast_pipeline.Association', to='vast_pipeline.Source')),
],
- ),
- migrations.AddField(
- model_name='measurement',
- name='source',
- field=models.ManyToManyField(through='vast_pipeline.Association', to='vast_pipeline.Source'),
+ options={
+ 'ordering': ['ra'],
+ },
),
migrations.AddField(
model_name='image',
@@ -319,45 +248,31 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='image',
name='skyreg',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.SkyRegion'),
- ),
- migrations.AddField(
- model_name='crossmatch',
- name='source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Source'),
- ),
- migrations.AddField(
- model_name='crossmatch',
- name='survey_source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.SurveySource'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.skyregion'),
),
migrations.CreateModel(
name='Comment',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('datetime', models.DateTimeField(auto_now_add=True)),
('comment', models.TextField()),
('object_id', models.PositiveIntegerField()),
('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
- ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+ ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
],
),
migrations.AddField(
model_name='association',
name='meas',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Measurement'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.measurement'),
),
migrations.AddField(
model_name='association',
name='source',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.Source'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vast_pipeline.source'),
),
migrations.AddConstraint(
model_name='relatedsource',
constraint=models.UniqueConstraint(fields=('from_source', 'to_source'), name='vast_pipeline_relatedsource_unique_pair'),
),
- migrations.AddConstraint(
- model_name='measurementpair',
- constraint=models.UniqueConstraint(fields=('source', 'measurement_a', 'measurement_b'), name='vast_pipeline_measurementpair_unique_pair'),
- ),
]
diff --git a/vast_pipeline/migrations/0002_q3c.py b/vast_pipeline/migrations/0002_q3c.py
deleted file mode 100644
index 54908b619..000000000
--- a/vast_pipeline/migrations/0002_q3c.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Generated by Django 2.2.5 on 2020-02-19 10:46
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- initial = False
-
- dependencies = [
- ('vast_pipeline', '0001_initial')
- ]
-
- operations = [
- migrations.RunSQL(
- ["CREATE EXTENSION IF NOT EXISTS q3c;"],#upgrade
- ["DROP EXTENSION IF EXISTS q3c;"],#downgrade
- ),
- migrations.RunSQL(
- ["CREATE INDEX ON vast_pipeline_measurement (q3c_ang2ipix(ra, dec));"],
- ["DROP INDEX vast_pipeline_measurement_q3c_ang2ipix_idx;"],
- ),
- migrations.RunSQL(
- ["CLUSTER vast_pipeline_measurement_q3c_ang2ipix_idx ON vast_pipeline_measurement;"],
- [],
- ),
- migrations.RunSQL(
- ["ANALYZE vast_pipeline_measurement;"],
- [],
- ),
- migrations.RunSQL(
- ["CREATE INDEX ON vast_pipeline_source (q3c_ang2ipix(wavg_ra, wavg_dec));"],
- ["DROP INDEX vast_pipeline_source_q3c_ang2ipix_idx;"],
- ),
- migrations.RunSQL(
- ["CLUSTER vast_pipeline_source_q3c_ang2ipix_idx ON vast_pipeline_source;"],
- [],
- ),
- migrations.RunSQL(
- ["ANALYZE vast_pipeline_source;"],
- [],
- ),
- ]
diff --git a/vast_pipeline/migrations/0003_auto_20210402_2136.py b/vast_pipeline/migrations/0003_auto_20210402_2136.py
deleted file mode 100644
index a8d4c3013..000000000
--- a/vast_pipeline/migrations/0003_auto_20210402_2136.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-02 21:36
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0002_q3c'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='source',
- name='m_abs_significant_max_int',
- field=models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for int flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.'),
- ),
- migrations.AlterField(
- model_name='source',
- name='m_abs_significant_max_peak',
- field=models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair modulation indices for peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.'),
- ),
- migrations.AlterField(
- model_name='source',
- name='vs_abs_significant_max_int',
- field=models.FloatField(default=0.0, help_text='Maximum value of all measurement pair variability t-statistics for int flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.'),
- ),
- migrations.AlterField(
- model_name='source',
- name='vs_abs_significant_max_peak',
- field=models.FloatField(default=0.0, help_text='Maximum absolute value of all measurement pair variability t-statistics for peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in the pipeline run configuration.'),
- ),
- ]
diff --git a/vast_pipeline/migrations/0004_auto_20210521_1339.py b/vast_pipeline/migrations/0004_auto_20210521_1339.py
deleted file mode 100644
index 01619928d..000000000
--- a/vast_pipeline/migrations/0004_auto_20210521_1339.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Generated by Django 3.2.2 on 2021-05-21 13:39
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0003_auto_20210402_2136'),
- ]
-
- operations = [
- migrations.RemoveField(
- model_name='surveysource',
- name='survey',
- ),
- migrations.RemoveField(
- model_name='source',
- name='cross_match_sources',
- ),
- migrations.DeleteModel(
- name='CrossMatch',
- ),
- migrations.DeleteModel(
- name='Survey',
- ),
- migrations.DeleteModel(
- name='SurveySource',
- ),
- ]
diff --git a/vast_pipeline/migrations/0004_source_name_change.py b/vast_pipeline/migrations/0004_source_name_change.py
deleted file mode 100644
index 56477a451..000000000
--- a/vast_pipeline/migrations/0004_source_name_change.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from django.db import migrations, transaction
-from vast_pipeline.utils.utils import deg2hms, deg2dms
-
-
-def update_source_names(apps, schema_editor):
- """Update source names from the v0.2.0 ASKAP_... convention to J..."""
- Source = apps.get_model("vast_pipeline", "Source")
- while Source.objects.filter(name__startswith="ASKAP_").exists():
- # do the updates in transaction batches of 1000 in case the source table is large
- with transaction.atomic():
- for source in Source.objects.filter(name__startswith="ASKAP_")[:1000]:
- source.name = (
- f"J{deg2hms(source.wavg_ra, precision=1)}"
- f"{deg2dms(source.wavg_dec, precision=0)}"
- ).replace(":", "")
- source.save()
-
-
-def reverse_update_source_names(apps, schema_editor):
- """Update source names from J... to the v0.2.0 ASKAP_... convention"""
- Source = apps.get_model("vast_pipeline", "Source")
- while Source.objects.filter(name__startswith="J").exists():
- # do the updates in transaction batches of 1000 in case the source table is large
- with transaction.atomic():
- for source in Source.objects.filter(name__startswith="J")[:1000]:
- source.name = (
- f"ASKAP_{deg2hms(source.wavg_ra, precision=2)}"
- f"{deg2dms(source.wavg_dec, precision=2)}"
- ).replace(":", "")
- source.save()
-
-
-class Migration(migrations.Migration):
- atomic = False # disable transactions, the source table may be large
-
- dependencies = [
- ("vast_pipeline", "0003_auto_20210402_2136"),
- ]
-
- operations = [
- migrations.RunPython(
- update_source_names, reverse_code=reverse_update_source_names
- ),
- ]
diff --git a/vast_pipeline/migrations/0005_merge_0004_auto_20210521_1339_0004_source_name_change.py b/vast_pipeline/migrations/0005_merge_0004_auto_20210521_1339_0004_source_name_change.py
deleted file mode 100644
index 4c318c4c9..000000000
--- a/vast_pipeline/migrations/0005_merge_0004_auto_20210521_1339_0004_source_name_change.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Generated by Django 3.2.2 on 2021-05-21 17:31
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0004_auto_20210521_1339'),
- ('vast_pipeline', '0004_source_name_change'),
- ]
-
- operations = [
- ]
diff --git a/vast_pipeline/migrations/0006_alter_run_status.py b/vast_pipeline/migrations/0006_alter_run_status.py
deleted file mode 100644
index 957f5faf0..000000000
--- a/vast_pipeline/migrations/0006_alter_run_status.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.2 on 2021-09-09 11:20
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0005_merge_0004_auto_20210521_1339_0004_source_name_change'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='run',
- name='status',
- field=models.CharField(choices=[('INI', 'Initialised'), ('QUE', 'Queued'), ('RUN', 'Running'), ('END', 'Completed'), ('ERR', 'Error'), ('RES', 'Restoring'), ('DEL', 'Deleting')], default='INI', help_text='Status of the pipeline run.', max_length=3),
- ),
- ]
diff --git a/vast_pipeline/migrations/0007_alter_measurement_name.py b/vast_pipeline/migrations/0007_alter_measurement_name.py
deleted file mode 100644
index b9f54e7b9..000000000
--- a/vast_pipeline/migrations/0007_alter_measurement_name.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.2 on 2021-10-11 20:07
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0006_alter_run_status'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='measurement',
- name='name',
- field=models.CharField(max_length=64),
- ),
- ]
diff --git a/vast_pipeline/migrations/0008_delete_measurementpair.py b/vast_pipeline/migrations/0008_delete_measurementpair.py
deleted file mode 100644
index f103feb54..000000000
--- a/vast_pipeline/migrations/0008_delete_measurementpair.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Generated by Django 3.2.9 on 2021-12-03 20:25
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0007_alter_measurement_name'),
- ]
-
- operations = [
- migrations.DeleteModel(
- name='MeasurementPair',
- ),
- ]
diff --git a/vast_pipeline/migrations/0009_add_log_file_timestamps.py b/vast_pipeline/migrations/0009_add_log_file_timestamps.py
deleted file mode 100644
index e086674b0..000000000
--- a/vast_pipeline/migrations/0009_add_log_file_timestamps.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Generated by Django 3.2.9 on 2022-01-14 13:30
-
-# This is a migration to rename the log files of old runs prior to when
-# they were timestamped. No DB changes occur in this migration.
-
-import shutil
-import warnings
-
-from django.db import migrations
-from pathlib import Path
-
-
-def get_timestamp(log_path: Path) -> str:
- """
- Opens the provided log file and extracts the timestamp from the
- first line.
-
- Args:
- log_path: The Path object for the log file.
-
- Returns:
- The string representation of the timestamp in the format of
- YYYY-MM-DD-HH-MM-SS.
- """
- with log_path.open() as f:
- line = f.readline()
-
- timestamp = (
- line.split(',')[0]
- .replace(' ', '-')
- .replace(':', '-')
- )
-
- return timestamp
-
-
-def add_timestamp_and_copy(log_path: Path) -> None:
- """
- Creates a new log file with the timestamp added to the name.
-
- Args:
- log_path: The Path object for the log file.
-
- Returns:
- None
- """
- timestamp = get_timestamp(log_path)
-
- new_log_name = f"{timestamp}_{log_path.name}"
- new_log_path = log_path.parent / new_log_name
-
- # check if already exists, pretty much impossible
- if new_log_path.exists():
- warnings.warn(
- f"Log file '{log_path}' not migrated as log file with timestamp"
- f" '{new_log_name}' already exists!"
- )
- return
-
- else:
- try:
- shutil.copy(log_path, new_log_path)
- except Exception as e:
- warnings.warn(
- f"Log file '{log_path}' could not be migrated as the copy"
- " operation failed. Please migrate this log file manually."
- f"Copy error: {e}."
- )
- return
-
- # a final check before deletion.
- if new_log_path.exists():
- log_path.unlink()
- else:
- warnings.warn(
- "While the copy operation was apparently successful the"
- f" new log file '{new_log_path}' does not exist. Please"
- f" migrate the old log file '{log_path}' manually."
- )
-
-
-def rename_logs(apps, schema_editor) -> None:
- """
- Loops through the existing runs and adds timestamps to the log files
- where required.
- """
- Run = apps.get_model('vast_pipeline', 'Run')
-
- # old names of logs before timestamping was introduced.
- logs_to_check = ['log.txt', 'restore_log.txt', 'gen_arrow_log.txt']
-
- for run in Run.objects.all():
- run_path = Path(run.path)
- for log in logs_to_check:
- log_path = run_path / log
- if log_path.exists():
- add_timestamp_and_copy(log_path)
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0008_delete_measurementpair'),
- ]
-
- operations = [
- migrations.RunPython(rename_logs),
- ]
diff --git a/vast_pipeline/migrations/0010_update_source_names_iau_compliant.py b/vast_pipeline/migrations/0010_update_source_names_iau_compliant.py
deleted file mode 100644
index d3aefdfb5..000000000
--- a/vast_pipeline/migrations/0010_update_source_names_iau_compliant.py
+++ /dev/null
@@ -1,68 +0,0 @@
-from django.db import migrations, transaction
-from vast_pipeline.utils.utils import deg2hms, deg2dms
-
-
-def _make_source_name(ra: float, dec: float, iau: bool = True) -> str:
- return (
- f"J{deg2hms(ra, precision=1, truncate=iau)}"
- f"{deg2dms(dec, precision=0, truncate=iau)}"
- ).replace(":", "")
-
-
-def update_source_names(apps, schema_editor):
- """Update source names that were generated by rounding coordinates instead of
- truncating. Truncating is required by IAU convention."""
- Source = apps.get_model("vast_pipeline", "Source")
-
- BATCH_SIZE = 1000
- sources = Source.objects.filter(name__startswith="J")
- start = 0
- end = start + BATCH_SIZE
- sources_batch = sources[start:end]
- while sources_batch.exists():
- with transaction.atomic():
- sources_batch_list = list(sources_batch)
- for source in sources_batch_list:
- source.name = _make_source_name(
- source.wavg_ra, source.wavg_dec, iau=True
- )
- _ = Source.objects.bulk_update(sources_batch_list, ["name"])
- start = end
- end = start + BATCH_SIZE
- sources_batch = sources[start:end]
-
-
-def reverse_update_source_names(apps, schema_editor):
- """Undo the above update source names by rounding coordinates instead of truncating."""
- Source = apps.get_model("vast_pipeline", "Source")
-
- BATCH_SIZE = 1000
- sources = Source.objects.filter(name__startswith="J")
- start = 0
- end = start + BATCH_SIZE
- sources_batch = sources[start:end]
- while sources_batch.exists():
- with transaction.atomic():
- sources_batch_list = list(sources_batch)
- for source in sources_batch_list:
- source.name = _make_source_name(
- source.wavg_ra, source.wavg_dec, iau=False
- )
- _ = Source.objects.bulk_update(sources_batch_list, ["name"])
- start = end
- end = start + BATCH_SIZE
- sources_batch = sources[start:end]
-
-
-class Migration(migrations.Migration):
- atomic = False
-
- dependencies = [
- ("vast_pipeline", "0009_add_log_file_timestamps"),
- ]
-
- operations = [
- migrations.RunPython(
- update_source_names, reverse_code=reverse_update_source_names, atomic=False
- ),
- ]
diff --git a/vast_pipeline/migrations/0011_run_n_new_sources.py b/vast_pipeline/migrations/0011_run_n_new_sources.py
deleted file mode 100644
index 44a42cd26..000000000
--- a/vast_pipeline/migrations/0011_run_n_new_sources.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Generated by Django 3.2.13 on 2023-01-25 18:13
-
-from django.db import migrations, models, transaction
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('vast_pipeline', '0010_update_source_names_iau_compliant'),
- ]
-
- def fill_new_source_counts(apps, schema_editor):
- """Loop over Run objects and count the number of new sources and save to new column."""
- Run = apps.get_model("vast_pipeline", "Run")
- Source = apps.get_model("vast_pipeline", "Source")
-
- runs = Run.objects.all()
-
- for run in runs:
- with transaction.atomic():
- n_new_sources = Source.objects.filter(run=run, new=True).count()
- run.n_new_sources = n_new_sources
- run.save()
-
- operations = [
- migrations.AddField(
- model_name='run',
- name='n_new_sources',
- field=models.IntegerField(default=0, help_text='number of new sources in this run'),
- ),
- migrations.RunPython(fill_new_source_counts, atomic=False)
- ]
diff --git a/vast_pipeline/models.py b/vast_pipeline/models.py
index 127a036b2..a4a52b8ae 100644
--- a/vast_pipeline/models.py
+++ b/vast_pipeline/models.py
@@ -1,3 +1,5 @@
+import uuid
+
from dataclasses import dataclass
from itertools import combinations
from pathlib import Path
@@ -31,12 +33,14 @@ class Comment(models.Model):
"""
The model object for a comment.
"""
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
author = models.ForeignKey(User, on_delete=models.CASCADE)
datetime = models.DateTimeField(auto_now_add=True)
comment = models.TextField()
content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
object_id = models.PositiveIntegerField()
- content_object = GenericForeignKey('content_type', 'object_id')
+ content_object = GenericForeignKey("content_type", "object_id")
def get_avatar_url(self) -> str:
"""Get the URL for the user's avatar from GitHub. If the user has
@@ -57,6 +61,7 @@ class CommentableModel(models.Model):
"""
A class to provide a commentable model.
"""
+
comment = GenericRelation(
Comment,
content_type_field="content_type",
@@ -69,7 +74,6 @@ class Meta:
class RunQuerySet(models.QuerySet):
-
def check_max_runs(self, max_runs: int = 5) -> int:
"""
Check if number of running pipeline runs is above threshold.
@@ -80,7 +84,7 @@ def check_max_runs(self, max_runs: int = 5) -> int:
Returns:
The count of the current pipeline runs with a status of `RUN`.
"""
- return self.filter(status='RUN').count() >= max_runs
+ return self.filter(status="RUN").count() >= max_runs
RunManager = models.Manager.from_queryset(RunQuerySet)
@@ -91,86 +95,73 @@ class Run(CommentableModel):
A Run is essentially a pipeline run/processing istance over a set of
images
"""
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(
- User,
- on_delete=models.SET_NULL,
- null=True,
- blank=True
+ User, on_delete=models.SET_NULL, null=True, blank=True, to_field="id"
)
-
name = models.CharField(
max_length=64,
unique=True,
validators=[
RegexValidator(
- regex=r'[\[@!#$%^&*()<>?/\|}{~:\] ]',
- message='Name contains not allowed characters!',
- inverse_match=True
+ regex=r"[\[@!#$%^&*()<>?/\|}{~:\] ]",
+ message="Name contains not allowed characters!",
+ inverse_match=True,
),
],
- help_text='name of the pipeline run'
+ help_text="name of the pipeline run",
)
description = models.CharField(
- max_length=240,
- blank=True,
- help_text="A short description of the pipeline run."
- )
- time = models.DateTimeField(
- auto_now=True,
- help_text='Datetime of a pipeline run.'
- )
- path = models.FilePathField(
- max_length=200,
- help_text='path to the pipeline run'
+ max_length=240, blank=True, help_text="A short description of the pipeline run."
)
+ time = models.DateTimeField(auto_now=True, help_text="Datetime of a pipeline run.")
+ path = models.FilePathField(max_length=200, help_text="path to the pipeline run")
STATUS_CHOICES = [
- ('INI', 'Initialised'),
- ('QUE', 'Queued'),
- ('RUN', 'Running'),
- ('END', 'Completed'),
- ('ERR', 'Error'),
- ('RES', 'Restoring'),
- ('DEL', 'Deleting'),
+ ("INI", "Initialised"),
+ ("QUE", "Queued"),
+ ("RUN", "Running"),
+ ("END", "Completed"),
+ ("ERR", "Error"),
+ ("RES", "Restoring"),
+ ("DEL", "Deleting"),
]
status = models.CharField(
max_length=3,
choices=STATUS_CHOICES,
- default='INI',
- help_text='Status of the pipeline run.'
+ default="INI",
+ help_text="Status of the pipeline run.",
)
n_images = models.IntegerField(
- default=0,
- help_text='number of images processed in this run'
+ default=0, help_text="number of images processed in this run"
)
n_sources = models.IntegerField(
- default=0,
- help_text='number of sources extracted in this run'
+ default=0, help_text="number of sources extracted in this run"
)
n_selavy_measurements = models.IntegerField(
- default=0,
- help_text='number of selavy measurements in this run'
+ default=0, help_text="number of selavy measurements in this run"
)
n_forced_measurements = models.IntegerField(
- default=0,
- help_text='number of forced measurements in this run'
+ default=0, help_text="number of forced measurements in this run"
)
n_new_sources = models.IntegerField(
- default=0,
- help_text='number of new sources in this run'
+ default=0, help_text="number of new sources in this run"
)
epoch_based = models.BooleanField(
default=False,
help_text=(
- 'Whether the run was processed using epoch based association'
- ', i.e. the user passed in groups of images defining epochs'
- ' rather than every image being treated individually.'
- )
+ "Whether the run was processed using epoch based association"
+ ", i.e. the user passed in groups of images defining epochs"
+ " rather than every image being treated individually."
+ ),
)
- objects = RunManager() # used instead of RunQuerySet.as_manager() so mypy checks work
+ objects = (
+ RunManager()
+ ) # used instead of RunQuerySet.as_manager() so mypy checks work
class Meta:
- ordering = ['name']
+ ordering = ["name"]
def __str__(self):
return self.name
@@ -211,22 +202,21 @@ class Band(models.Model):
A band on the frequency spectrum used for imaging. Each image is
associated with one band.
"""
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
name = models.CharField(max_length=12, unique=True)
- frequency = models.FloatField(
- help_text='central frequency of band (integer MHz)'
- )
- bandwidth = models.FloatField(
- help_text='bandwidth (MHz)'
- )
+ frequency = models.FloatField(help_text="central frequency of band (integer MHz)")
+ bandwidth = models.FloatField(help_text="bandwidth (MHz)")
class Meta:
- ordering = ['frequency']
+ ordering = ["frequency"]
def __str__(self):
return self.name
class SkyRegion(models.Model):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
run = models.ManyToManyField(Run)
centre_ra = models.FloatField()
@@ -239,14 +229,11 @@ class SkyRegion(models.Model):
z = models.FloatField()
def __str__(self):
- return f'{round(self.centre_ra, 3)}, {round(self.centre_dec, 3)}'
+ return f"{round(self.centre_ra, 3)}, {round(self.centre_dec, 3)}"
class SourceQuerySet(models.QuerySet):
-
- def cone_search(
- self, ra: float, dec: float, radius_deg: float
- ) -> models.QuerySet:
+ def cone_search(self, ra: float, dec: float, radius_deg: float) -> models.QuerySet:
"""
Return all the Sources withing radius_deg of (ra,dec).
Returns a QuerySet of Sources, ordered by distance from
@@ -261,131 +248,101 @@ def cone_search(
Returns:
Sources found withing the cone search area.
"""
- return (
- self.extra(
- select={
- "distance": "q3c_dist(wavg_ra, wavg_dec, %s, %s) * 3600"
- },
- select_params=[ra, dec],
- where=["q3c_radial_query(wavg_ra, wavg_dec, %s, %s, %s)"],
- params=[ra, dec, radius_deg],
- )
- .order_by("distance")
- )
+ return self.extra(
+ select={"distance": "q3c_dist(wavg_ra, wavg_dec, %s, %s) * 3600"},
+ select_params=[ra, dec],
+ where=["q3c_radial_query(wavg_ra, wavg_dec, %s, %s, %s)"],
+ params=[ra, dec, radius_deg],
+ ).order_by("distance")
class Image(CommentableModel):
"""An image is a 2D radio image from a cube"""
- band = models.ForeignKey(Band, on_delete=models.CASCADE)
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ band = models.ForeignKey(Band, on_delete=models.CASCADE, to_field="id")
run = models.ManyToManyField(Run)
- skyreg = models.ForeignKey(SkyRegion, on_delete=models.CASCADE)
+ skyreg = models.ForeignKey(SkyRegion, on_delete=models.CASCADE, to_field="id")
measurements_path = models.FilePathField(
max_length=200,
- db_column='meas_path',
- help_text=(
- 'the path to the measurements parquet that belongs to this image'
- )
+ db_column="meas_path",
+ help_text=("the path to the measurements parquet that belongs to this image"),
)
POLARISATION_CHOICES = [
- ('I', 'I'),
- ('XX', 'XX'),
- ('YY', 'YY'),
- ('Q', 'Q'),
- ('U', 'U'),
- ('V', 'V'),
+ ("I", "I"),
+ ("XX", "XX"),
+ ("YY", "YY"),
+ ("Q", "Q"),
+ ("U", "U"),
+ ("V", "V"),
]
polarisation = models.CharField(
max_length=2,
choices=POLARISATION_CHOICES,
- help_text='Polarisation of the image one of I,XX,YY,Q,U,V.'
- )
- name = models.CharField(
- unique=True,
- max_length=200,
- help_text='Name of the image.'
+ help_text="Polarisation of the image one of I,XX,YY,Q,U,V.",
)
+ name = models.CharField(unique=True, max_length=200, help_text="Name of the image.")
path = models.FilePathField(
- max_length=500,
- help_text='Path to the file containing the image.'
+ max_length=500, help_text="Path to the file containing the image."
)
noise_path = models.FilePathField(
max_length=300,
blank=True,
- default='',
- help_text='Path to the file containing the RMS image.'
+ default="",
+ help_text="Path to the file containing the RMS image.",
)
background_path = models.FilePathField(
max_length=300,
blank=True,
- default='',
- help_text='Path to the file containing the background image.'
+ default="",
+ help_text="Path to the file containing the background image.",
)
- datetime = models.DateTimeField(
- help_text='Date/time of observation or epoch.'
- )
- jd = models.FloatField(
- help_text='Julian date of the observation (days).'
- )
- duration = models.FloatField(
- default=0.,
- help_text='Duration of the observation.'
- )
+ datetime = models.DateTimeField(help_text="Date/time of observation or epoch.")
+ jd = models.FloatField(help_text="Julian date of the observation (days).")
+ duration = models.FloatField(default=0.0, help_text="Duration of the observation.")
- ra = models.FloatField(
- help_text='RA of the image centre (Deg).'
- )
- dec = models.FloatField(
- help_text='DEC of the image centre (Deg).'
- )
+ ra = models.FloatField(help_text="RA of the image centre (Deg).")
+ dec = models.FloatField(help_text="DEC of the image centre (Deg).")
fov_bmaj = models.FloatField(
- help_text='Field of view major axis (Deg).'
+ help_text="Field of view major axis (Deg)."
) # Major (Dec) radius of image (degrees)
fov_bmin = models.FloatField(
- help_text='Field of view minor axis (Deg).'
+ help_text="Field of view minor axis (Deg)."
) # Minor (RA) radius of image (degrees)
physical_bmaj = models.FloatField(
- help_text='The actual size of the image major axis (Deg).'
+ help_text="The actual size of the image major axis (Deg)."
) # Major (Dec) radius of image (degrees)
physical_bmin = models.FloatField(
- help_text='The actual size of the image minor axis (Deg).'
+ help_text="The actual size of the image minor axis (Deg)."
) # Minor (RA) radius of image (degrees)
radius_pixels = models.FloatField(
- help_text='Radius of the useable region of the image (pixels).'
+ help_text="Radius of the useable region of the image (pixels)."
)
- beam_bmaj = models.FloatField(
- help_text='Major axis of image restoring beam (Deg).'
- )
- beam_bmin = models.FloatField(
- help_text='Minor axis of image restoring beam (Deg).'
- )
- beam_bpa = models.FloatField(
- help_text='Beam position angle (Deg).'
- )
+ beam_bmaj = models.FloatField(help_text="Major axis of image restoring beam (Deg).")
+ beam_bmin = models.FloatField(help_text="Minor axis of image restoring beam (Deg).")
+ beam_bpa = models.FloatField(help_text="Beam position angle (Deg).")
rms_median = models.FloatField(
- help_text='Background average RMS from the provided RMS map (mJy).'
+ help_text="Background average RMS from the provided RMS map (mJy)."
)
rms_min = models.FloatField(
- help_text='Background minimum RMS from the provided RMS map (mJy).'
+ help_text="Background minimum RMS from the provided RMS map (mJy)."
)
rms_max = models.FloatField(
- help_text='Background maximum RMS from the provided RMS map (mJy).'
+ help_text="Background maximum RMS from the provided RMS map (mJy)."
)
class Meta:
- ordering = ['datetime']
+ ordering = ["datetime"]
def __str__(self):
return self.name
class MeasurementQuerySet(models.QuerySet):
-
- def cone_search(
- self, ra: float, dec: float, radius_deg: float
- ) -> models.QuerySet:
+ def cone_search(self, ra: float, dec: float, radius_deg: float) -> models.QuerySet:
"""
Return all the Sources withing radius_deg of (ra,dec).
Returns a QuerySet of Sources, ordered by distance from
@@ -400,17 +357,12 @@ def cone_search(
Returns:
Measurements found withing the cone search area.
"""
- return (
- self.extra(
- select={
- "distance": "q3c_dist(ra, dec, %s, %s) * 3600"
- },
- select_params=[ra, dec],
- where=["q3c_radial_query(ra, dec, %s, %s, %s)"],
- params=[ra, dec, radius_deg],
- )
- .order_by("distance")
- )
+ return self.extra(
+ select={"distance": "q3c_dist(ra, dec, %s, %s) * 3600"},
+ select_params=[ra, dec],
+ where=["q3c_radial_query(ra, dec, %s, %s, %s)"],
+ params=[ra, dec, radius_deg],
+ ).order_by("distance")
class Measurement(CommentableModel):
@@ -418,78 +370,64 @@ class Measurement(CommentableModel):
A Measurement is an object in the sky that has been detected at least once.
Essentially a source single measurement in time.
"""
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
image = models.ForeignKey(
- Image,
- null=True,
- on_delete=models.CASCADE
+ Image, null=True, on_delete=models.CASCADE, to_field="id"
) # first image seen in
source = models.ManyToManyField(
- 'Source',
- through='Association',
- through_fields=('meas', 'source')
+ "Source", through="Association", through_fields=("meas", "source")
)
name = models.CharField(max_length=64)
- ra = models.FloatField(help_text='RA of the source (Deg).') # degrees
- ra_err = models.FloatField(
- help_text='RA error of the source (Deg).'
- )
- dec = models.FloatField(help_text='DEC of the source (Deg).') # degrees
- dec_err = models.FloatField(
- help_text='DEC error of the source (Deg).'
- )
+ ra = models.FloatField(help_text="RA of the source (Deg).") # degrees
+ ra_err = models.FloatField(help_text="RA error of the source (Deg).")
+ dec = models.FloatField(help_text="DEC of the source (Deg).") # degrees
+ dec_err = models.FloatField(help_text="DEC error of the source (Deg).")
bmaj = models.FloatField(
- help_text=(
- 'The major axis of the Gaussian fit to the source (Deg).'
- )
+ help_text=("The major axis of the Gaussian fit to the source (Deg).")
)
- err_bmaj = models.FloatField(help_text='Error major axis (Deg).')
+ err_bmaj = models.FloatField(help_text="Error major axis (Deg).")
bmin = models.FloatField(
- help_text=(
- 'The minor axis of the Gaussian fit to the source (Deg).'
- )
+ help_text=("The minor axis of the Gaussian fit to the source (Deg).")
)
- err_bmin = models.FloatField(help_text='Error minor axis (Deg).')
+ err_bmin = models.FloatField(help_text="Error minor axis (Deg).")
pa = models.FloatField(
- help_text=(
- 'Position angle of Gaussian fit east of north to bmaj '
- '(Deg).'
- )
+ help_text=("Position angle of Gaussian fit east of north to bmaj " "(Deg).")
)
- err_pa = models.FloatField(help_text='Error position angle (Deg).')
+ err_pa = models.FloatField(help_text="Error position angle (Deg).")
# supplied by user via config
ew_sys_err = models.FloatField(
- help_text='Systematic error in east-west (RA) direction (Deg).'
+ help_text="Systematic error in east-west (RA) direction (Deg)."
)
# supplied by user via config
ns_sys_err = models.FloatField(
- help_text='Systematic error in north-south (dec) direction (Deg).'
+ help_text="Systematic error in north-south (dec) direction (Deg)."
)
# estimate of maximum error radius (from ra_err and dec_err)
# Used in advanced association.
error_radius = models.FloatField(
help_text=(
- 'Estimate of maximum error radius using ra_err'
- ' and dec_err (Deg).'
+ "Estimate of maximum error radius using ra_err" " and dec_err (Deg)."
)
)
# quadratic sum of error_radius and ew_sys_err
uncertainty_ew = models.FloatField(
help_text=(
- 'Total east-west (RA) uncertainty, quadratic sum of'
- ' error_radius and ew_sys_err (Deg).'
+ "Total east-west (RA) uncertainty, quadratic sum of"
+ " error_radius and ew_sys_err (Deg)."
)
)
# quadratic sum of error_radius and ns_sys_err
uncertainty_ns = models.FloatField(
help_text=(
- 'Total north-south (Dec) uncertainty, quadratic sum of '
- 'error_radius and ns_sys_err (Deg).'
+ "Total north-south (Dec) uncertainty, quadratic sum of "
+ "error_radius and ns_sys_err (Deg)."
)
)
@@ -497,94 +435,73 @@ class Measurement(CommentableModel):
flux_int_err = models.FloatField() # mJy/beam
flux_int_isl_ratio = models.FloatField(
help_text=(
- 'Ratio of the component integrated flux to the total'
- ' island integrated flux.'
+ "Ratio of the component integrated flux to the total"
+ " island integrated flux."
)
)
flux_peak = models.FloatField() # mJy/beam
flux_peak_err = models.FloatField() # mJy/beam
flux_peak_isl_ratio = models.FloatField(
- help_text=(
- 'Ratio of the component peak flux to the total'
- ' island peak flux.'
- )
+ help_text=("Ratio of the component peak flux to the total" " island peak flux.")
)
chi_squared_fit = models.FloatField(
- db_column='chi2_fit',
- help_text='Chi-squared of the Guassian fit to the source.'
+ db_column="chi2_fit", help_text="Chi-squared of the Guassian fit to the source."
)
spectral_index = models.FloatField(
- db_column='spectr_idx',
- help_text='In-band Selavy spectral index.'
+ db_column="spectr_idx", help_text="In-band Selavy spectral index."
)
spectral_index_from_TT = models.BooleanField(
default=False,
- db_column='spectr_idx_tt',
- help_text=(
- 'True/False if the spectral index came from the taylor '
- 'term.'
- )
+ db_column="spectr_idx_tt",
+ help_text=("True/False if the spectral index came from the taylor " "term."),
)
- local_rms = models.FloatField(
- help_text='Local rms in mJy from Selavy.'
- ) # mJy/beam
+ local_rms = models.FloatField(help_text="Local rms in mJy from Selavy.") # mJy/beam
- snr = models.FloatField(
- help_text='Signal-to-noise ratio of the measurement.'
- )
+ snr = models.FloatField(help_text="Signal-to-noise ratio of the measurement.")
- flag_c4 = models.BooleanField(
- default=False,
- help_text='Fit flag from Selavy.'
- )
+ flag_c4 = models.BooleanField(default=False, help_text="Fit flag from Selavy.")
- compactness = models.FloatField(
- help_text='Int flux over peak flux.'
- )
+ compactness = models.FloatField(help_text="Int flux over peak flux.")
has_siblings = models.BooleanField(
default=False,
- help_text='True if the fit comes from an island that has more than 1 component.'
+ help_text="True if the fit comes from an island that has more than 1 component.",
)
component_id = models.CharField(
max_length=64,
- help_text=(
- 'The ID of the component from which the source comes from.'
- )
+ help_text=("The ID of the component from which the source comes from."),
)
island_id = models.CharField(
max_length=64,
- help_text=(
- 'The ID of the island from which the source comes from.'
- )
+ help_text=("The ID of the island from which the source comes from."),
)
forced = models.BooleanField(
- default=False,
- help_text='True: the measurement is forced extracted.'
+ default=False, help_text="True: the measurement is forced extracted."
)
objects = MeasurementQuerySet.as_manager()
class Meta:
- ordering = ['ra']
+ ordering = ["ra"]
def __str__(self):
return self.name
class Source(CommentableModel):
- run = models.ForeignKey(Run, on_delete=models.CASCADE, null=True,)
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ run = models.ForeignKey(Run, on_delete=models.CASCADE, null=True, to_field="id")
related = models.ManyToManyField(
- 'self',
- through='RelatedSource',
+ "self",
+ through="RelatedSource",
symmetrical=False,
- through_fields=('from_source', 'to_source')
+ through_fields=("from_source", "to_source"),
)
name = models.CharField(max_length=100)
- new = models.BooleanField(default=False, help_text='New Source.')
+ new = models.BooleanField(default=False, help_text="New Source.")
tags = TagField(
space_delimiter=False,
autocomplete_view="vast_pipeline:source_tags_autocomplete",
@@ -592,129 +509,98 @@ class Source(CommentableModel):
)
# average fields calculated from the source measurements
- wavg_ra = models.FloatField(
- help_text='The weighted average right ascension (Deg).'
- )
- wavg_dec = models.FloatField(
- help_text='The weighted average declination (Deg).'
- )
+ wavg_ra = models.FloatField(help_text="The weighted average right ascension (Deg).")
+ wavg_dec = models.FloatField(help_text="The weighted average declination (Deg).")
wavg_uncertainty_ew = models.FloatField(
help_text=(
- 'The weighted average uncertainty in the east-'
- 'west (RA) direction (Deg).'
+ "The weighted average uncertainty in the east-" "west (RA) direction (Deg)."
)
)
wavg_uncertainty_ns = models.FloatField(
help_text=(
- 'The weighted average uncertainty in the north-'
- 'south (Dec) direction (Deg).'
+ "The weighted average uncertainty in the north-"
+ "south (Dec) direction (Deg)."
)
)
- avg_flux_int = models.FloatField(
- help_text='The average integrated flux value.'
- )
- avg_flux_peak = models.FloatField(
- help_text='The average peak flux value.'
- )
- max_flux_peak = models.FloatField(
- help_text='The maximum peak flux value.'
- )
- min_flux_peak = models.FloatField(
- help_text='The minimum peak flux value.'
- )
- max_flux_int = models.FloatField(
- help_text='The maximum integrated flux value.'
- )
- min_flux_int = models.FloatField(
- help_text='The minimum integrated flux value.'
- )
+ avg_flux_int = models.FloatField(help_text="The average integrated flux value.")
+ avg_flux_peak = models.FloatField(help_text="The average peak flux value.")
+ max_flux_peak = models.FloatField(help_text="The maximum peak flux value.")
+ min_flux_peak = models.FloatField(help_text="The minimum peak flux value.")
+ max_flux_int = models.FloatField(help_text="The maximum integrated flux value.")
+ min_flux_int = models.FloatField(help_text="The minimum integrated flux value.")
min_flux_int_isl_ratio = models.FloatField(
- help_text='The minimum integrated island flux ratio value.'
+ help_text="The minimum integrated island flux ratio value."
)
min_flux_peak_isl_ratio = models.FloatField(
- help_text='The minimum peak island flux ratio value.'
- )
- avg_compactness = models.FloatField(
- help_text='The average compactness.'
+ help_text="The minimum peak island flux ratio value."
)
+ avg_compactness = models.FloatField(help_text="The average compactness.")
min_snr = models.FloatField(
- help_text='The minimum signal-to-noise ratio value of the detections.'
+ help_text="The minimum signal-to-noise ratio value of the detections."
)
max_snr = models.FloatField(
- help_text='The maximum signal-to-noise ratio value of the detections.'
+ help_text="The maximum signal-to-noise ratio value of the detections."
)
# metrics
- v_int = models.FloatField(
- help_text='V metric for int flux.'
- )
- v_peak = models.FloatField(
- help_text='V metric for peak flux.'
- )
- eta_int = models.FloatField(
- help_text='Eta metric for int flux.'
- )
- eta_peak = models.FloatField(
- help_text='Eta metric for peak flux.'
- )
+ v_int = models.FloatField(help_text="V metric for int flux.")
+ v_peak = models.FloatField(help_text="V metric for peak flux.")
+ eta_int = models.FloatField(help_text="Eta metric for int flux.")
+ eta_peak = models.FloatField(help_text="Eta metric for peak flux.")
new_high_sigma = models.FloatField(
help_text=(
- 'The largest sigma value for the new source'
- ' if it was placed in previous image.'
+ "The largest sigma value for the new source"
+ " if it was placed in previous image."
)
)
n_neighbour_dist = models.FloatField(
- help_text='Distance to the nearest neighbour (deg)'
+ help_text="Distance to the nearest neighbour (deg)"
)
vs_abs_significant_max_int = models.FloatField(
default=0.0,
help_text=(
- 'Maximum value of all measurement pair variability t-statistics for int'
- ' flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in'
- ' the pipeline run configuration.'
- )
+ "Maximum value of all measurement pair variability t-statistics for int"
+ " flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in"
+ " the pipeline run configuration."
+ ),
)
m_abs_significant_max_int = models.FloatField(
default=0.0,
help_text=(
- 'Maximum absolute value of all measurement pair modulation indices for int'
- ' flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in'
- ' the pipeline run configuration.'
- )
+ "Maximum absolute value of all measurement pair modulation indices for int"
+ " flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs in"
+ " the pipeline run configuration."
+ ),
)
vs_abs_significant_max_peak = models.FloatField(
default=0.0,
help_text=(
- 'Maximum absolute value of all measurement pair variability t-statistics for'
- ' peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs'
- ' in the pipeline run configuration.'
- )
+ "Maximum absolute value of all measurement pair variability t-statistics for"
+ " peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs"
+ " in the pipeline run configuration."
+ ),
)
m_abs_significant_max_peak = models.FloatField(
default=0.0,
help_text=(
- 'Maximum absolute value of all measurement pair modulation indices for '
- ' peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs'
- ' in the pipeline run configuration.'
- )
+ "Maximum absolute value of all measurement pair modulation indices for "
+ " peak flux that exceed variability.source_aggregate_pair_metrics_min_abs_vs"
+ " in the pipeline run configuration."
+ ),
)
# total metrics to report in UI
- n_meas = models.IntegerField(
- help_text='total measurements of the source'
- )
+ n_meas = models.IntegerField(help_text="total measurements of the source")
n_meas_sel = models.IntegerField(
- help_text='total selavy extracted measurements of the source'
+ help_text="total selavy extracted measurements of the source"
)
n_meas_forced = models.IntegerField(
- help_text='total force extracted measurements of the source'
+ help_text="total force extracted measurements of the source"
)
n_rel = models.IntegerField(
- help_text='total relations of the source with other sources'
- )
- n_sibl = models.IntegerField(
- help_text='total siblings of the source'
+ help_text="total relations of the source with other sources"
)
+ n_sibl = models.IntegerField(help_text="total siblings of the source")
objects = SourceQuerySet.as_manager()
@@ -749,15 +635,23 @@ def get_measurement_pairs(self) -> List[MeasurementPair]:
meas_a, meas_b = meas_b, meas_a
# calculate metrics
vs_peak = calculate_vs_metric(
- meas_a.flux_peak, meas_b.flux_peak, meas_a.flux_peak_err, meas_b.flux_peak_err
+ meas_a.flux_peak,
+ meas_b.flux_peak,
+ meas_a.flux_peak_err,
+ meas_b.flux_peak_err,
)
m_int = calculate_m_metric(meas_a.flux_int, meas_b.flux_int)
vs_int = calculate_vs_metric(
- meas_a.flux_int, meas_b.flux_int, meas_a.flux_int_err, meas_b.flux_int_err
+ meas_a.flux_int,
+ meas_b.flux_int,
+ meas_a.flux_int_err,
+ meas_b.flux_int_err,
)
m_peak = calculate_m_metric(meas_a.flux_peak, meas_b.flux_peak)
measurement_pairs.append(
- MeasurementPair(self.id, meas_a.id, meas_b.id, vs_peak, m_peak, vs_int, m_int)
+ MeasurementPair(
+ self.id, meas_a.id, meas_b.id, vs_peak, m_peak, vs_int, m_int
+ )
)
return measurement_pairs
@@ -767,54 +661,54 @@ class Association(models.Model):
model association between sources and measurements based on
some parameters
"""
- source = models.ForeignKey(Source, on_delete=models.CASCADE)
- meas = models.ForeignKey(Measurement, on_delete=models.CASCADE)
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ source = models.ForeignKey(Source, on_delete=models.CASCADE, to_field="id")
+ meas = models.ForeignKey(Measurement, on_delete=models.CASCADE, to_field="id")
d2d = models.FloatField(
- default=0.,
- help_text='astronomical distance calculated by Astropy, arcsec.'
+ default=0.0, help_text="astronomical distance calculated by Astropy, arcsec."
)
dr = models.FloatField(
- default=0.,
- help_text='De Ruiter radius calculated in advanced association.'
+ default=0.0, help_text="De Ruiter radius calculated in advanced association."
)
def __str__(self):
return (
- f'distance: {self.d2d:.2f}' if self.dr == 0 else
- f'distance: {self.dr:.2f}'
+ f"distance: {self.d2d:.2f}" if self.dr == 0 else f"distance: {self.dr:.2f}"
)
class RelatedSource(models.Model):
- '''
+ """
Association table for the many to many Source relationship with itself
Django doc
https://docs.djangoproject.com/en/3.1/ref/models/fields/#django.db.models.ManyToManyField.through
- '''
- from_source = models.ForeignKey(Source, on_delete=models.CASCADE)
+ """
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ from_source = models.ForeignKey(Source, on_delete=models.CASCADE, to_field="id")
to_source = models.ForeignKey(
- Source,
- on_delete=models.CASCADE,
- related_name='related_sources'
+ Source, on_delete=models.CASCADE, related_name="related_sources", to_field="id"
)
class Meta:
constraints = [
models.UniqueConstraint(
- name='%(app_label)s_%(class)s_unique_pair',
- fields=['from_source', 'to_source']
+ name="%(app_label)s_%(class)s_unique_pair",
+ fields=["from_source", "to_source"],
)
]
class SourceFav(models.Model):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, on_delete=models.CASCADE)
- source = models.ForeignKey(Source, on_delete=models.CASCADE)
+ source = models.ForeignKey(Source, on_delete=models.CASCADE, to_field="id")
comment = models.TextField(
max_length=500,
- default='',
+ default="",
blank=True,
- help_text='Why did you include this as favourite'
+ help_text="Why did you include this as favourite",
)
diff --git a/vast_pipeline/pipeline/association.py b/vast_pipeline/pipeline/association.py
index b66940ce7..f86b73be9 100644
--- a/vast_pipeline/pipeline/association.py
+++ b/vast_pipeline/pipeline/association.py
@@ -1,8 +1,8 @@
"""
This module contains all the functions required to perform source association.
"""
-
import logging
+import uuid
import numpy as np
import pandas as pd
from typing import Tuple, Dict, List
@@ -17,7 +17,7 @@
prep_skysrc_df,
add_new_one_to_many_relations,
add_new_many_to_one_relations,
- reconstruct_associtaion_dfs
+ reconstruct_associtaion_dfs,
)
from vast_pipeline.pipeline.config import PipelineConfig
from vast_pipeline.utils.utils import StopWatch
@@ -43,29 +43,29 @@ def calc_de_ruiter(df: pd.DataFrame) -> np.ndarray:
Returns:
Array containing the de Ruiter radius for all rows in the df.
"""
- ra_1 = df['ra_skyc1'].values
- ra_2 = df['ra_skyc2'].values
+ ra_1 = df["ra_skyc1"].values
+ ra_2 = df["ra_skyc2"].values
# avoid wrapping issues
- ra_1[ra_1 > 270.] -= 180.
- ra_2[ra_2 > 270.] -= 180.
- ra_1[ra_1 < 90.] += 180.
- ra_2[ra_2 < 90.] += 180.
+ ra_1[ra_1 > 270.0] -= 180.0
+ ra_2[ra_2 > 270.0] -= 180.0
+ ra_1[ra_1 < 90.0] += 180.0
+ ra_2[ra_2 < 90.0] += 180.0
ra_1 = np.deg2rad(ra_1)
ra_2 = np.deg2rad(ra_2)
- ra_1_err = np.deg2rad(df['uncertainty_ew_skyc1'].values)
- ra_2_err = np.deg2rad(df['uncertainty_ew_skyc2'].values)
+ ra_1_err = np.deg2rad(df["uncertainty_ew_skyc1"].values)
+ ra_2_err = np.deg2rad(df["uncertainty_ew_skyc2"].values)
- dec_1 = np.deg2rad(df['dec_skyc1'].values)
- dec_2 = np.deg2rad(df['dec_skyc2'].values)
+ dec_1 = np.deg2rad(df["dec_skyc1"].values)
+ dec_2 = np.deg2rad(df["dec_skyc2"].values)
- dec_1_err = np.deg2rad(df['uncertainty_ns_skyc1'].values)
- dec_2_err = np.deg2rad(df['uncertainty_ns_skyc2'].values)
+ dec_1_err = np.deg2rad(df["uncertainty_ns_skyc1"].values)
+ dec_2_err = np.deg2rad(df["uncertainty_ns_skyc2"].values)
dr1 = (ra_1 - ra_2) * (ra_1 - ra_2)
- dr1_1 = np.cos((dec_1 + dec_2) / 2.)
+ dr1_1 = np.cos((dec_1 + dec_2) / 2.0)
dr1 *= dr1_1 * dr1_1
dr1 /= ra_1_err * ra_1_err + ra_2_err * ra_2_err
@@ -78,9 +78,7 @@ def calc_de_ruiter(df: pd.DataFrame) -> np.ndarray:
def one_to_many_basic(
- skyc2_srcs: pd.DataFrame,
- sources_df: pd.DataFrame,
- id_incr_par_assoc: int = 0
+ skyc2_srcs: pd.DataFrame, sources_df: pd.DataFrame
) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
Finds and processes the one-to-many associations in the basic
@@ -98,9 +96,6 @@ def one_to_many_basic(
associated to the base) used during basic association.
sources_df: The sources_df produced by each step of
association holding the current 'sources'.
- id_incr_par_assoc: An increment value to add to new source ids
- when creating them. Mainly useful for add mode with parallel
- association
Returns:
Updated 'skyc2_srcs' with all the one_to_many relation information
@@ -110,9 +105,8 @@ def one_to_many_basic(
"""
# select duplicated in 'source' field in skyc2_srcs, excluding -1
duplicated_skyc2 = skyc2_srcs.loc[
- (skyc2_srcs['source'] != -1) &
- skyc2_srcs['source'].duplicated(keep=False),
- ['source', 'related', 'd2d']
+ (skyc2_srcs["source"].notnull()) & skyc2_srcs["source"].duplicated(keep=False),
+ ["source", "related", "d2d"],
]
# duplicated_skyc2
@@ -127,13 +121,10 @@ def one_to_many_basic(
# +-----+----------+-----------+---------+
if duplicated_skyc2.empty:
- logger.debug('No one-to-many associations.')
+ logger.debug("No one-to-many associations.")
return skyc2_srcs, sources_df
- logger.info(
- 'Detected #%i double matches, cleaning...',
- duplicated_skyc2.shape[0]
- )
+ logger.info("Detected #%i double matches, cleaning...", duplicated_skyc2.shape[0])
# now we have the src values which are doubled.
# make the nearest match have the "original" src id
@@ -141,27 +132,20 @@ def one_to_many_basic(
# and make sure to copy the other previously
# matched sources.
# Get the duplicated, sort by the distance column
- duplicated_skyc2 = duplicated_skyc2.sort_values(by=['source', 'd2d'])
+ duplicated_skyc2 = duplicated_skyc2.sort_values(by=["source", "d2d"])
- # Get those that need to be given a new ID number (i.e. not the min dist_col)
- idx_to_change = duplicated_skyc2.index.values[
- duplicated_skyc2.duplicated('source')
+ # Get those that need to be given a new ID (i.e. not the min dist_col)
+ idx_to_change = duplicated_skyc2.index.to_numpy()[
+ duplicated_skyc2.duplicated("source")
]
# Create a new `new_source_id` column to store the 'correct' IDs
- duplicated_skyc2['new_source_id'] = duplicated_skyc2['source']
-
- # Define the range of new source ids
- start_new_src_id = sources_df['source'].values.max() + 1 + id_incr_par_assoc
+ duplicated_skyc2["new_source_id"] = duplicated_skyc2["source"]
- new_source_ids = np.arange(
- start_new_src_id,
- start_new_src_id + idx_to_change.shape[0],
- dtype=int
- )
+ new_source_ids = [str(uuid.uuid4()) for _ in range(idx_to_change.shape[0])]
# Assign the new IDs
- duplicated_skyc2.loc[idx_to_change, 'new_source_id'] = new_source_ids
+ duplicated_skyc2.loc[idx_to_change, "new_source_id"] = new_source_ids
# duplicated_skyc2
# +-----+----------+-----------+---------+-----------------+
@@ -180,20 +164,14 @@ def one_to_many_basic(
# related column.
# The not_original ones need just the original ID appended.
# copy() is used here to avoid chained indexing (set with copy warnings)
- not_original = duplicated_skyc2.loc[
- idx_to_change
- ].copy()
+ not_original = duplicated_skyc2.loc[idx_to_change].copy()
- original = duplicated_skyc2.drop_duplicates(
- 'source'
- ).copy()
+ original = duplicated_skyc2.drop_duplicates("source").copy()
new_original_related = pd.DataFrame(
- not_original[
- ['source', 'new_source_id']
- ].groupby('source').apply(
- lambda grp: grp['new_source_id'].tolist()
- )
+ not_original[["source", "new_source_id"]]
+ .groupby("source")
+ .apply(lambda grp: grp["new_source_id"].tolist())
)
# new_original_related
@@ -210,19 +188,12 @@ def one_to_many_basic(
# Append the relations in each case, using the above 'new_original_related'
# for the original ones.
# The not original only require the appending of the original index.
- original['related'] = (
- original[['related', 'source']]
- .apply(
- add_new_one_to_many_relations,
- args=(False, new_original_related),
- axis=1
- )
+ original["related"] = original[["related", "source"]].apply(
+ add_new_one_to_many_relations, args=(False, new_original_related), axis=1
)
- not_original['related'] = not_original.apply(
- add_new_one_to_many_relations,
- args=(False,),
- axis=1
+ not_original["related"] = not_original.apply(
+ add_new_one_to_many_relations, args=(False,), axis=1
)
duplicated_skyc2 = pd.concat([original, not_original])
@@ -241,31 +212,23 @@ def one_to_many_basic(
del original, not_original
# Apply the updates to the actual temp_srcs.
- skyc2_srcs.loc[idx_to_change, 'source'] = new_source_ids
- skyc2_srcs.loc[
- duplicated_skyc2.index.values,
- 'related'
- ] = duplicated_skyc2.loc[
- duplicated_skyc2.index.values,
- 'related'
+ skyc2_srcs.loc[idx_to_change, "source"] = new_source_ids
+ skyc2_srcs.loc[duplicated_skyc2.index.values, "related"] = duplicated_skyc2.loc[
+ duplicated_skyc2.index.values, "related"
].values
# Finally we need to copy copies of the previous sources in the
# sources_df to complete the new sources.
# To do this we get only the non-original sources
- duplicated_skyc2 = duplicated_skyc2.loc[
- duplicated_skyc2.duplicated('source')
- ]
+ duplicated_skyc2 = duplicated_skyc2.loc[duplicated_skyc2.duplicated("source")]
# Get all the indexes required for each original
# `source_skyc1` value
source_df_index_to_copy = pd.DataFrame(
- duplicated_skyc2.groupby(
- 'source'
- ).apply(
+ duplicated_skyc2.groupby("source").apply(
lambda grp: sources_df[
- sources_df['source'] == grp.name
+ sources_df["source"] == grp.name
].index.values.tolist()
)
)
@@ -283,33 +246,23 @@ def one_to_many_basic(
# merge these so it's easy to explode and copy the index values.
duplicated_skyc2 = (
- duplicated_skyc2[['source', 'new_source_id']]
- .merge(
- source_df_index_to_copy,
- left_on='source',
- right_index=True,
- how='left'
- )
- .rename(columns={0: 'source_index'})
- .explode('source_index')
+ duplicated_skyc2[["source", "new_source_id"]]
+ .merge(source_df_index_to_copy, left_on="source", right_index=True, how="left")
+ .rename(columns={0: "source_index"})
+ .explode("source_index")
)
# Get the sources - all columns from the sources_df table
- sources_to_copy = sources_df.loc[
- duplicated_skyc2['source_index'].values
- ]
+ sources_to_copy = sources_df.loc[duplicated_skyc2["source_index"].values]
# Apply the new_source_id
- sources_to_copy['source'] = duplicated_skyc2['new_source_id'].values
+ sources_to_copy["source"] = duplicated_skyc2["new_source_id"].values
# Reset the related column to avoid rogue relations
- sources_to_copy['related'] = None
+ sources_to_copy["related"] = None
# and finally concatenate.
- sources_df = pd.concat(
- [sources_df, sources_to_copy],
- ignore_index=True
- )
+ sources_df = pd.concat([sources_df, sources_to_copy], ignore_index=True)
return skyc2_srcs, sources_df
@@ -318,9 +271,8 @@ def one_to_many_advanced(
temp_srcs: pd.DataFrame,
sources_df: pd.DataFrame,
method: str,
- id_incr_par_assoc: int = 0
) -> Tuple[pd.DataFrame, pd.DataFrame]:
- '''
+ """
Finds and processes the one-to-many associations in the advanced
association. For each one-to-many association, the nearest
associated source is assigned the original source id, where as
@@ -341,24 +293,27 @@ def one_to_many_advanced(
method:
Can be either 'advanced' or 'deruiter' to represent the advanced
association method being used.
- id_incr_par_assoc:
- An increment value to add to new source ids when creating them.
- Mainly useful for add mode with parallel association
Returns:
Updated `temp_srcs` dataframe with all the one_to_many relation
information added.
Updated `sources_df` dataframe with all the one_to_many relation
information added.
- '''
+ """
# use only these columns for easy debugging of the dataframe
cols = [
- 'index_old_skyc1', 'id_skyc1', 'source_skyc1',
- 'related_skyc1', 'index_old_skyc2', 'id_skyc2', 'source_skyc2',
- 'd2d', 'dr'
+ "index_old_skyc1",
+ "id_skyc1",
+ "source_skyc1",
+ "related_skyc1",
+ "index_old_skyc2",
+ "id_skyc2",
+ "source_skyc2",
+ "d2d",
+ "dr",
]
duplicated_skyc1 = temp_srcs.loc[
- temp_srcs['source_skyc1'].duplicated(keep=False), cols
+ temp_srcs["source_skyc1"].duplicated(keep=False), cols
].copy()
# duplicated_skyc1
@@ -392,18 +347,17 @@ def one_to_many_advanced(
# If no relations then no action is required
if duplicated_skyc1.empty:
- logger.debug('No one-to-many associations.')
+ logger.debug("No one-to-many associations.")
return temp_srcs, sources_df
logger.debug(
- 'Detected #%i one-to-many assocations, cleaning...',
- duplicated_skyc1.shape[0]
+ "Detected #%i one-to-many assocations, cleaning...", duplicated_skyc1.shape[0]
)
# Get the column to check for the minimum depending on the method
# set the column names needed for filtering the 'to-many'
# associations depending on the method (advanced or deruiter)
- dist_col = 'd2d' if method == 'advanced' else 'dr'
+ dist_col = "d2d" if method == "advanced" else "dr"
# go through the doubles and
# 1. Keep the closest d2d or de ruiter as the primary id
@@ -413,17 +367,15 @@ def one_to_many_advanced(
# multi_srcs = duplicated_skyc1['source_skyc1'].unique()
# Get the duplicated, sort by the distance column
- duplicated_skyc1 = duplicated_skyc1.sort_values(
- by=['source_skyc1', dist_col]
- )
+ duplicated_skyc1 = duplicated_skyc1.sort_values(by=["source_skyc1", dist_col])
# Get those that need to be given a new ID number (i.e. not the min dist_col)
idx_to_change = duplicated_skyc1.index.values[
- duplicated_skyc1.duplicated('source_skyc1')
+ duplicated_skyc1.duplicated("source_skyc1")
]
# Create a new `new_source_id` column to store the 'correct' IDs
- duplicated_skyc1['new_source_id'] = duplicated_skyc1['source_skyc1']
+ duplicated_skyc1["new_source_id"] = duplicated_skyc1["source_skyc1"]
# +-----------------+
# | new_source_id |
@@ -435,47 +387,34 @@ def one_to_many_advanced(
# | 262 |
# +-----------------+
- # Define the range of new source ids
- start_new_src_id = sources_df['source'].values.max() + 1 + id_incr_par_assoc
-
# Create an arange to use to change the ones that need to be changed.
- new_source_ids = np.arange(
- start_new_src_id,
- start_new_src_id + idx_to_change.shape[0],
- dtype=int
- )
+ new_source_ids = [str(uuid.uuid4()) for _ in range(idx_to_change.shape[0])]
# Assign the new IDs to those that need to be changed.
- duplicated_skyc1.loc[idx_to_change, 'new_source_id'] = new_source_ids
+ duplicated_skyc1.loc[idx_to_change, "new_source_id"] = new_source_ids
# We also need to clear the relations for these 'new' sources
# otherwise it will inherit rogue relations from the original relation
- duplicated_skyc1.loc[idx_to_change, 'related_skyc1'] = None
+ duplicated_skyc1.loc[idx_to_change, "related_skyc1"] = None
# Now we need to sort out the related, essentially here the 'original'
# and 'non original' need to be treated differently.
- # The original source need all the assoicated new ids appended to the
+ # The original source need all the associated new ids appended to the
# related column.
# The not_original ones need just the original ID appended.
- not_original = duplicated_skyc1.loc[
- idx_to_change
- ].copy()
+ not_original = duplicated_skyc1.loc[idx_to_change].copy()
- original = duplicated_skyc1.drop_duplicates(
- 'source_skyc1'
- ).copy()
+ original = duplicated_skyc1.drop_duplicates("source_skyc1").copy()
# This gathers all the new ids that need to be appended
# to the original related column.
new_original_related = pd.DataFrame(
- not_original[
- ['source_skyc1', 'new_source_id']
- ].groupby('source_skyc1').apply(
- lambda grp: grp['new_source_id'].tolist()
- )
+ not_original[["source_skyc1", "new_source_id"]]
+ .groupby("source_skyc1")
+ .apply(lambda grp: grp["new_source_id"].tolist())
)
- #new_original_related
+ # new_original_related
# +----------------+--------+
# | source_skyc1 | 0 |
# |----------------+--------|
@@ -489,13 +428,8 @@ def one_to_many_advanced(
# Append the relations in each case, using the above 'new_original_related'
# for the original ones.
# The not original only require the appending of the original index.
- original['related_skyc1'] = (
- original[['related_skyc1', 'source_skyc1']]
- .apply(
- add_new_one_to_many_relations,
- args=(True, new_original_related),
- axis=1
- )
+ original["related_skyc1"] = original[["related_skyc1", "source_skyc1"]].apply(
+ add_new_one_to_many_relations, args=(True, new_original_related), axis=1
)
# what the column looks like after the above
@@ -509,10 +443,8 @@ def one_to_many_advanced(
# | [5546] |
# +-----------------+
- not_original.loc[:, 'related_skyc1'] = not_original.apply(
- add_new_one_to_many_relations,
- args=(True,),
- axis=1
+ not_original.loc[:, "related_skyc1"] = not_original.apply(
+ add_new_one_to_many_relations, args=(True,), axis=1
)
# Merge them back together
@@ -521,31 +453,23 @@ def one_to_many_advanced(
del original, not_original
# Apply the updates to the actual temp_srcs.
- temp_srcs.loc[idx_to_change, 'source_skyc1'] = new_source_ids
+ temp_srcs.loc[idx_to_change, "source_skyc1"] = new_source_ids
temp_srcs.loc[
- duplicated_skyc1.index.values,
- 'related_skyc1'
- ] = duplicated_skyc1.loc[
- duplicated_skyc1.index.values,
- 'related_skyc1'
- ].values
+ duplicated_skyc1.index.values, "related_skyc1"
+ ] = duplicated_skyc1.loc[duplicated_skyc1.index.values, "related_skyc1"].values
# Finally we need to create copies of the previous sources in the
# sources_df to complete the new sources.
# To do this we get only the non-original sources
- duplicated_skyc1 = duplicated_skyc1.loc[
- duplicated_skyc1.duplicated('source_skyc1')
- ]
+ duplicated_skyc1 = duplicated_skyc1.loc[duplicated_skyc1.duplicated("source_skyc1")]
# Get all the indexes required for each original
# `source_skyc1` value
source_df_index_to_copy = pd.DataFrame(
- duplicated_skyc1.groupby(
- 'source_skyc1'
- ).apply(
+ duplicated_skyc1.groupby("source_skyc1").apply(
lambda grp: sources_df[
- sources_df['source'] == grp.name
+ sources_df["source"] == grp.name
].index.values.tolist()
)
)
@@ -563,15 +487,15 @@ def one_to_many_advanced(
# merge these so it's easy to explode and copy the index values.
duplicated_skyc1 = (
- duplicated_skyc1.loc[:,['source_skyc1', 'new_source_id']]
+ duplicated_skyc1.loc[:, ["source_skyc1", "new_source_id"]]
.merge(
source_df_index_to_copy,
- left_on='source_skyc1',
+ left_on="source_skyc1",
right_index=True,
- how='left'
+ how="left",
)
- .rename(columns={0: 'source_index'})
- .explode('source_index')
+ .rename(columns={0: "source_index"})
+ .explode("source_index")
)
# duplicated_skyc1
@@ -586,15 +510,13 @@ def one_to_many_advanced(
# +-----+----------------+-----------------+----------------+
# Get the sources
- sources_to_copy = sources_df.loc[
- duplicated_skyc1['source_index'].values
- ]
+ sources_to_copy = sources_df.loc[duplicated_skyc1["source_index"].values]
# Apply the new_source_id
- sources_to_copy['source'] = duplicated_skyc1['new_source_id'].values
+ sources_to_copy["source"] = duplicated_skyc1["new_source_id"].values
# Reset the related column to avoid rogue relations
- sources_to_copy['related'] = None
+ sources_to_copy["related"] = None
# and finally concatenate.
sources_df = pd.concat([sources_df, sources_to_copy], ignore_index=True)
@@ -603,7 +525,7 @@ def one_to_many_advanced(
def many_to_many_advanced(temp_srcs: pd.DataFrame, method: str) -> pd.DataFrame:
- '''
+ """
Finds and processes the many-to-many associations in the advanced
association. We do not want to build many-to-many associations as
this will make the database get very large (see TraP documentation).
@@ -625,33 +547,29 @@ def many_to_many_advanced(temp_srcs: pd.DataFrame, method: str) -> pd.DataFrame:
Returns:
Updated temp_srcs with the many_to_many relations dropped.
- '''
+ """
# Select those where the extracted source is listed more than once
# (e.g. index_old_skyc2 duplicated values) and of these get those that
# have a source id that is listed more than once (e.g. source_skyc1
# duplicated values) in the temps_srcs df
- m_to_m = temp_srcs[(
- temp_srcs['index_old_skyc2'].duplicated(keep=False) &
- temp_srcs['source_skyc1'].duplicated(keep=False)
- )].copy()
+ m_to_m = temp_srcs[
+ (
+ temp_srcs["index_old_skyc2"].duplicated(keep=False)
+ & temp_srcs["source_skyc1"].duplicated(keep=False)
+ )
+ ].copy()
if m_to_m.empty:
- logger.debug('No many-to-many assocations.')
+ logger.debug("No many-to-many assocations.")
return temp_srcs
- logger.debug(
- 'Detected #%i many-to-many assocations, cleaning...',
- m_to_m.shape[0]
- )
+ logger.debug("Detected #%i many-to-many assocations, cleaning...", m_to_m.shape[0])
- dist_col = 'd2d' if method == 'advanced' else 'dr'
- min_col = 'min_' + dist_col
+ dist_col = "d2d" if method == "advanced" else "dr"
+ min_col = "min_" + dist_col
# get the minimum de ruiter value for each extracted source
- m_to_m[min_col] = (
- m_to_m.groupby('index_old_skyc2')[dist_col]
- .transform('min')
- )
+ m_to_m[min_col] = m_to_m.groupby("index_old_skyc2")[dist_col].transform("min")
# get the ids of those crossmatches that are larger than the minimum
m_to_m_to_drop = m_to_m[m_to_m[dist_col] != m_to_m[min_col]].index.values
# and drop these from the temp_srcs
@@ -661,7 +579,7 @@ def many_to_many_advanced(temp_srcs: pd.DataFrame, method: str) -> pd.DataFrame:
def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame:
- '''
+ """
Finds and processes the many-to-one associations in the advanced
association. In this case in the related column of the 'many' sources
we need to append the ids of all the other 'many' (expect for itself).
@@ -673,19 +591,24 @@ def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame:
Returns:
Updated temp_srcs with all many_to_one relation information added.
- '''
+ """
# use only these columns for easy debugging of the dataframe
cols = [
- 'index_old_skyc1', 'id_skyc1', 'source_skyc1',
- 'related_skyc1', 'index_old_skyc2', 'id_skyc2', 'source_skyc2',
- 'd2d', 'dr'
+ "index_old_skyc1",
+ "id_skyc1",
+ "source_skyc1",
+ "related_skyc1",
+ "index_old_skyc2",
+ "id_skyc2",
+ "source_skyc2",
+ "d2d",
+ "dr",
]
# select those sources which have been matched to the same measurement
# in the sky catalogue 2.
duplicated_skyc2 = temp_srcs.loc[
- temp_srcs['index_old_skyc2'].duplicated(keep=False),
- cols
+ temp_srcs["index_old_skyc2"].duplicated(keep=False), cols
]
# duplicated_skyc2
@@ -719,23 +642,20 @@ def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame:
# if there are none no action is required.
if duplicated_skyc2.empty:
- logger.debug('No many-to-one associations.')
+ logger.debug("No many-to-one associations.")
return temp_srcs
- logger.debug(
- 'Detected #%i many-to-one associations',
- duplicated_skyc2.shape[0]
- )
+ logger.debug("Detected #%i many-to-one associations", duplicated_skyc2.shape[0])
# The new relations become that for each 'many' source we need to append
# the ids of the other 'many' sources that have been associationed with the
# 'one'. Below for each 'one' group we gather all the ids of the many
# sources.
new_relations = pd.DataFrame(
- duplicated_skyc2
- .groupby('index_old_skyc2')
- .apply(lambda grp: grp['source_skyc1'].tolist())
- ).rename(columns={0: 'new_relations'})
+ duplicated_skyc2.groupby("index_old_skyc2").apply(
+ lambda grp: grp["source_skyc1"].tolist()
+ )
+ ).rename(columns={0: "new_relations"})
# new_relations
# +-------------------+-----------------+
@@ -751,52 +671,40 @@ def many_to_one_advanced(temp_srcs: pd.DataFrame) -> pd.DataFrame:
# these new relations are then added to the duplciated dataframe so
# they can easily be used by the next function.
duplicated_skyc2 = duplicated_skyc2.merge(
- new_relations,
- left_on='index_old_skyc2',
- right_index=True,
- how='left'
+ new_relations, left_on="index_old_skyc2", right_index=True, how="left"
)
# Remove the 'self' relations. The 'x['source_skyc1']' is an integer so it
# is placed within a list notation, [], to be able to be easily subtracted
# from the new_relations.
- duplicated_skyc2['new_relations'] = (
- duplicated_skyc2.apply(
- lambda x: list(set(x['new_relations']) - set([x['source_skyc1']])),
- axis=1
- )
+ duplicated_skyc2["new_relations"] = duplicated_skyc2.apply(
+ lambda x: list(set(x["new_relations"]) - set([x["source_skyc1"]])), axis=1
)
# Use the 'add_new_many_to_one_relations' method to add tthe new relatitons
# to the actual `related_skyc1' column.
- duplicated_skyc2['related_skyc1'] = (
- duplicated_skyc2.apply(
- add_new_many_to_one_relations,
- axis=1
- )
+ duplicated_skyc2["related_skyc1"] = duplicated_skyc2.apply(
+ add_new_many_to_one_relations, axis=1
)
# Transfer the new relations from the duplicated df to the temp_srcs. The
# index is explicitly declared to avoid any mixups.
temp_srcs.loc[
- duplicated_skyc2.index.values, 'related_skyc1'
- ] = duplicated_skyc2.loc[
- duplicated_skyc2.index.values, 'related_skyc1'
- ].values
+ duplicated_skyc2.index.values, "related_skyc1"
+ ] = duplicated_skyc2.loc[duplicated_skyc2.index.values, "related_skyc1"].values
return temp_srcs
def basic_association(
- sources_df: pd.DataFrame,
- skyc1_srcs: pd.DataFrame,
- skyc1: SkyCoord,
- skyc2_srcs: pd.DataFrame,
- skyc2: SkyCoord,
- limit: Angle,
- id_incr_par_assoc: int = 0
- ) -> Tuple[pd.DataFrame, pd.DataFrame]:
- '''
+ sources_df: pd.DataFrame,
+ skyc1_srcs: pd.DataFrame,
+ skyc1: SkyCoord,
+ skyc2_srcs: pd.DataFrame,
+ skyc2: SkyCoord,
+ limit: Angle,
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+ """
The loop for basic source association that uses the astropy
'match_to_catalog_sky' function (i.e. only the nearest match between
the catalogs). A direct on sky separation is used to define the association.
@@ -819,81 +727,63 @@ def basic_association(
A SkyCoord object with the sky positions from skyc2_srcs.
limit:
The association limit to use (applies to basic and advanced only).
- id_incr_par_assoc:
- An increment value to be applied to source numbering when adding
- new sources to the associations (applies when parallel and add
- image are being used). Defaults to 0.
Returns:
The output `sources_df` containing all input measurements along with the
association and relation information.
The output `skyc1_srcs` with updated with new sources from the
association.
- '''
+ """
# match the new sources to the base
# idx gives the index of the closest match in the base for skyc2
- idx, d2d, d3d = skyc2.match_to_catalog_sky(skyc1)
+ idx, d2d, _ = skyc2.match_to_catalog_sky(skyc1)
# acceptable selection
sel = d2d <= limit
# The good matches can be assinged the src id from base
- skyc2_srcs.loc[sel, 'source'] = skyc1_srcs.loc[idx[sel], 'source'].values
+ skyc2_srcs.loc[sel, "source"] = skyc1_srcs.loc[idx[sel], "source"].values
# Need the d2d to make analysing doubles easier.
- skyc2_srcs.loc[sel, 'd2d'] = d2d[sel].arcsec
+ skyc2_srcs.loc[sel, "d2d"] = d2d[sel].arcsec
# must check for double matches in the acceptable matches just made
# this would mean that multiple sources in skyc2 have been matched
# to the same base source we want to keep closest match and move
# the other match(es) back to having a -1 src id
- skyc2_srcs, sources_df = one_to_many_basic(
- skyc2_srcs, sources_df, id_incr_par_assoc)
+ skyc2_srcs, sources_df = one_to_many_basic(skyc2_srcs, sources_df)
- logger.info('Updating sources catalogue with new sources...')
+ logger.info("Updating sources catalogue with new sources...")
# update the src numbers for those sources in skyc2 with no match
- # using the max current src as the start and incrementing by one
- start_elem = sources_df['source'].values.max() + 1 + id_incr_par_assoc
- nan_sel = (skyc2_srcs['source'] == -1).values
- skyc2_srcs.loc[nan_sel, 'source'] = (
- np.arange(
- start_elem,
- start_elem + skyc2_srcs.loc[nan_sel].shape[0],
- dtype=int
- )
- )
+ nan_sel = (skyc2_srcs["source"].isnull()).to_numpy()
+ skyc2_srcs.loc[nan_sel, "source"] = [
+ str(uuid.uuid4()) for _ in range(nan_sel.sum())
+ ]
# and skyc2 is now ready to be concatenated with the new sources
- sources_df = pd.concat(
- [sources_df, skyc2_srcs],
- ignore_index=True
- ).reset_index(drop=True)
+ sources_df = pd.concat([sources_df, skyc2_srcs], ignore_index=True).reset_index(
+ drop=True
+ )
# and update skyc1 with the sources that were created from the one
# to many relations and any new sources.
skyc1_srcs = pd.concat(
- [
- skyc1_srcs,
- skyc2_srcs[
- ~skyc2_srcs['source'].isin(skyc1_srcs['source'])
- ]
- ],
- ignore_index=True
+ [skyc1_srcs, skyc2_srcs[~skyc2_srcs["source"].isin(skyc1_srcs["source"])]],
+ ignore_index=True,
).reset_index(drop=True)
return sources_df, skyc1_srcs
def advanced_association(
- method: str,
- sources_df: pd.DataFrame,
- skyc1_srcs: pd.DataFrame,
- skyc1: SkyCoord,
- skyc2_srcs: pd.DataFrame,
- skyc2: SkyCoord,
- dr_limit: float,
- bw_max: float,
- id_incr_par_assoc: int = 0
- ) -> Tuple[pd.DataFrame, pd.DataFrame]:
- '''
+ method: str,
+ sources_df: pd.DataFrame,
+ skyc1_srcs: pd.DataFrame,
+ skyc1: SkyCoord,
+ skyc2_srcs: pd.DataFrame,
+ skyc2: SkyCoord,
+ dr_limit: float,
+ bw_max: float,
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+ """
The loop for advanced source association that uses the astropy
'search_around_sky' function (i.e. all matching sources are
found). The BMAJ of the image * the user supplied beamwidth
@@ -922,48 +812,36 @@ def advanced_association(
The de Ruiter radius limit to use (applies to de ruiter only).
bw_max:
The beamwidth limit to use (applies to de ruiter only).
- id_incr_par_assoc:
- An increment value to be applied to source numbering when adding
- new sources to the associations (applies when parallel and add
- image are being used). Defaults to 0.
Returns:
The output `sources_df` containing all input measurements along with the
association and relation information.
The output `skyc1_srcs` with updated with new sources from the
association.
- '''
+ """
# read the needed sources fields
# Step 1: get matches within semimajor axis of image.
- idx_skyc1, idx_skyc2, d2d, d3d = skyc2.search_around_sky(
- skyc1, bw_max
- )
+ idx_skyc1, idx_skyc2, d2d, d3d = skyc2.search_around_sky(skyc1, bw_max)
# Step 2: merge the candidates so the de ruiter can be calculated
temp_skyc1_srcs = (
- skyc1_srcs.loc[idx_skyc1]
- .reset_index()
- .rename(columns={'index': 'index_old'})
+ skyc1_srcs.loc[idx_skyc1].reset_index().rename(columns={"index": "index_old"})
)
temp_skyc2_srcs = (
- skyc2_srcs.loc[idx_skyc2]
- .reset_index()
- .rename(columns={'index': 'index_old'})
+ skyc2_srcs.loc[idx_skyc2].reset_index().rename(columns={"index": "index_old"})
)
- temp_skyc2_srcs['d2d'] = d2d.arcsec
+ temp_skyc2_srcs["d2d"] = d2d.arcsec
temp_srcs = temp_skyc1_srcs.merge(
temp_skyc2_srcs,
left_index=True,
right_index=True,
- suffixes=('_skyc1', '_skyc2')
+ suffixes=("_skyc1", "_skyc2"),
)
# drop the double d2d column and keep the d2d_skyc2 as assigned above
- temp_srcs = (
- temp_srcs
- .drop(['d2d_skyc1', 'dr_skyc1', 'dr_skyc2'], axis=1)
- .rename(columns={'d2d_skyc2': 'd2d'})
+ temp_srcs = temp_srcs.drop(["d2d_skyc1", "dr_skyc1", "dr_skyc2"], axis=1).rename(
+ columns={"d2d_skyc2": "d2d"}
)
del temp_skyc1_srcs, temp_skyc2_srcs
@@ -972,11 +850,11 @@ def advanced_association(
temp_srcs = temp_srcs[d2d <= bw_max].copy()
# Step 4: Calculate and perform De Ruiter radius cut
- if method == 'deruiter':
- temp_srcs['dr'] = calc_de_ruiter(temp_srcs)
- temp_srcs = temp_srcs[temp_srcs['dr'] <= dr_limit]
+ if method == "deruiter":
+ temp_srcs["dr"] = calc_de_ruiter(temp_srcs)
+ temp_srcs = temp_srcs[temp_srcs["dr"] <= dr_limit]
else:
- temp_srcs['dr'] = 0.
+ temp_srcs["dr"] = 0.0
# Now have the 'good' matches
# Step 5: Check for one-to-many, many-to-one and many-to-many
@@ -986,7 +864,7 @@ def advanced_association(
# Next one-to-many
# Get the sources which are doubled
temp_srcs, sources_df = one_to_many_advanced(
- temp_srcs, sources_df, method, id_incr_par_assoc
+ temp_srcs, sources_df, method
)
# Finally many-to-one associations, the opposite of above but we
@@ -999,47 +877,35 @@ def advanced_association(
# This is created from the temp_srcs df.
# This will take care of the extra skyc2 sources needed.
skyc2_srcs_toappend = skyc2_srcs.loc[
- temp_srcs['index_old_skyc2'].values
+ temp_srcs["index_old_skyc2"].values
].reset_index(drop=True)
- skyc2_srcs_toappend['source'] = temp_srcs['source_skyc1'].values
- skyc2_srcs_toappend['related'] = temp_srcs['related_skyc1'].values
- skyc2_srcs_toappend['d2d'] = temp_srcs['d2d'].values
- skyc2_srcs_toappend['dr'] = temp_srcs['dr'].values
+ skyc2_srcs_toappend["source"] = temp_srcs["source_skyc1"].values
+ skyc2_srcs_toappend["related"] = temp_srcs["related_skyc1"].values
+ skyc2_srcs_toappend["d2d"] = temp_srcs["d2d"].values
+ skyc2_srcs_toappend["dr"] = temp_srcs["dr"].values
# and get the skyc2 sources with no match
- logger.info(
- 'Updating sources catalogue with new sources...'
- )
+ logger.info("Updating sources catalogue with new sources...")
new_sources = skyc2_srcs.loc[
- skyc2_srcs.index.difference(
- temp_srcs['index_old_skyc2'].values
- )
+ skyc2_srcs.index.difference(temp_srcs["index_old_skyc2"].values)
].reset_index(drop=True)
# update the src numbers for those sources in skyc2 with no match
# using the max current src as the start and incrementing by one
- start_elem = sources_df['source'].values.max() + 1 + id_incr_par_assoc
- new_sources['source'] = np.arange(
- start_elem,
- start_elem + new_sources.shape[0],
- dtype=int
- )
+ new_sources["source"] = new_sources.apply(lambda _: str(uuid.uuid4()), axis=1)
skyc2_srcs_toappend = pd.concat(
- [skyc2_srcs_toappend, new_sources],
- ignore_index=True
+ [skyc2_srcs_toappend, new_sources], ignore_index=True
)
# and skyc2 is now ready to be concatenated with source_df
sources_df = pd.concat(
- [sources_df, skyc2_srcs_toappend],
- ignore_index=True
+ [sources_df, skyc2_srcs_toappend], ignore_index=True
).reset_index(drop=True)
# update skyc1 and df for next association iteration
# calculate average angles for skyc1
- skyc1_srcs = pd.concat(
- [skyc1_srcs, new_sources],
- ignore_index=True
- ).reset_index(drop=True)
+ skyc1_srcs = pd.concat([skyc1_srcs, new_sources], ignore_index=True).reset_index(
+ drop=True
+ )
# also need to append any related sources that created a new
# source, we can use the skyc2_srcs_toappend to get these
@@ -1048,7 +914,7 @@ def advanced_association(
skyc1_srcs,
skyc2_srcs_toappend.loc[
~skyc2_srcs_toappend.source.isin(skyc1_srcs.source)
- ]
+ ],
]
)
@@ -1065,10 +931,9 @@ def association(
add_mode: bool,
previous_parquets: Dict[str, str],
done_images_df: pd.DataFrame,
- id_incr_par_assoc: int = 0,
- parallel: bool = False
+ parallel: bool = False,
) -> pd.DataFrame:
- '''
+ """
The main association function that does the common tasks between basic
and advanced modes.
@@ -1094,10 +959,6 @@ def association(
done_images_df:
Datafraame containing the images of the previous successful run
(used in add image mode).
- id_incr_par_assoc:
- An increment value to be applied to source numbering when adding
- new sources to the associations (applies when parallel and add
- image are being used). Defaults to 0.
parallel:
Whether parallel association is being used.
@@ -1107,7 +968,7 @@ def association(
Raises:
Exception: Raised if association method is not valid.
- '''
+ """
timer = StopWatch()
if parallel:
@@ -1116,13 +977,12 @@ def association(
if len(images_df) == 0:
return images_df
- images_df = (
- images_df.sort_values(by='image_datetime')
- .drop('image_datetime', axis=1)
+ images_df = images_df.sort_values(by="image_datetime").drop(
+ "image_datetime", axis=1
)
- if 'skyreg_group' in images_df.columns:
- skyreg_group = images_df['skyreg_group'].iloc[0]
+ if "skyreg_group" in images_df.columns:
+ skyreg_group = images_df["skyreg_group"].iloc[0]
skyreg_tag = " (sky region group %s)" % skyreg_group
else:
skyreg_group = -1
@@ -1130,15 +990,15 @@ def association(
method = config["source_association"]["method"]
- logger.info('Starting association%s.', skyreg_tag)
- logger.info('Association mode selected: %s.', method)
+ logger.info("Starting association%s.", skyreg_tag)
+ logger.info("Association mode selected: %s.", method)
- unique_epochs = np.sort(images_df['epoch'].unique())
+ unique_epochs = np.sort(images_df["epoch"].unique())
if add_mode:
# Here the skyc1_srcs and sources_df are recreated and the done images
# are filtered out.
- image_mask = images_df['image_name'].isin(done_images_df['name'])
+ image_mask = images_df["image_name"].isin(done_images_df["name"])
images_df_done = images_df[image_mask].copy()
sources_df, skyc1_srcs = reconstruct_associtaion_dfs(
images_df_done,
@@ -1146,42 +1006,35 @@ def association(
)
images_df = images_df.loc[~image_mask]
if images_df.empty:
- logger.info(
- 'No new images found, stopping association%s.', skyreg_tag
- )
- sources_df['interim_ew'] = (
- sources_df['ra_source'].values * sources_df['weight_ew'].values
+ logger.info("No new images found, stopping association%s.", skyreg_tag)
+ sources_df["interim_ew"] = (
+ sources_df["ra_source"].values * sources_df["weight_ew"].values
)
- sources_df['interim_ns'] = (
- sources_df['dec_source'].values * sources_df['weight_ns'].values
+ sources_df["interim_ns"] = (
+ sources_df["dec_source"].values * sources_df["weight_ns"].values
)
- return (
- sources_df
- .drop(['ra', 'dec'], axis=1)
- .rename(columns={'ra_source': 'ra', 'dec_source': 'dec'})
+ return sources_df.drop(["ra", "dec"], axis=1).rename(
+ columns={"ra_source": "ra", "dec_source": "dec"}
)
- logger.info(
- f'Found {images_df.shape[0]} images to add to the run{skyreg_tag}.')
+ logger.info(f"Found {images_df.shape[0]} images to add to the run{skyreg_tag}.")
# re-get the unique epochs
- unique_epochs = np.sort(images_df['epoch'].unique())
+ unique_epochs = np.sort(images_df["epoch"].unique())
start_epoch = 0
else:
# Do full set up for a new run.
- first_images = (
- images_df
- .loc[images_df['epoch'] == unique_epochs[0], 'image_dj']
- .to_list()
- )
+ first_images = images_df.loc[
+ images_df["epoch"] == unique_epochs[0], "image_dj"
+ ].to_list()
# initialise sky source dataframe
skyc1_srcs = prep_skysrc_df(
first_images,
config["measurements"]["flux_fractional_error"],
duplicate_limit,
- ini_df=True
+ ini_df=True,
)
- skyc1_srcs['epoch'] = unique_epochs[0]
+ skyc1_srcs["epoch"] = unique_epochs[0]
# create base catalogue
# initialise the sources dataframe using first image as base
sources_df = skyc1_srcs.copy()
@@ -1193,60 +1046,71 @@ def association(
# images to be added, the interim needs to be calculated and skyc1_srcs
# can just be returned as sources_df. ra_source and dec_source can just
# be dropped as the ra and dec are already the average values.
- logger.warning(
- 'No images to associate with!%s.', skyreg_tag
- )
- logger.info(
- 'Returning base sources only%s.', skyreg_tag
- )
+ logger.warning("No images to associate with!%s.", skyreg_tag)
+ logger.info("Returning base sources only%s.", skyreg_tag)
# reorder the columns to match Dask expectations (parallel)
- skyc1_srcs = skyc1_srcs[[
- 'id', 'uncertainty_ew', 'weight_ew', 'uncertainty_ns', 'weight_ns',
- 'flux_int', 'flux_int_err', 'flux_int_isl_ratio', 'flux_peak',
- 'flux_peak_err', 'flux_peak_isl_ratio', 'forced', 'compactness',
- 'has_siblings', 'snr', 'image', 'datetime', 'source', 'ra', 'dec',
- 'ra_source', 'dec_source', 'd2d', 'dr', 'related', 'epoch',
- ]]
- skyc1_srcs['interim_ew'] = (
- skyc1_srcs['ra'].values * skyc1_srcs['weight_ew'].values
+ skyc1_srcs = skyc1_srcs[
+ [
+ "id",
+ "uncertainty_ew",
+ "weight_ew",
+ "uncertainty_ns",
+ "weight_ns",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "forced",
+ "compactness",
+ "has_siblings",
+ "snr",
+ "image",
+ "datetime",
+ "source",
+ "ra",
+ "dec",
+ "ra_source",
+ "dec_source",
+ "d2d",
+ "dr",
+ "related",
+ "epoch",
+ ]
+ ]
+ skyc1_srcs["interim_ew"] = (
+ skyc1_srcs["ra"].values * skyc1_srcs["weight_ew"].values
)
- skyc1_srcs['interim_ns'] = (
- skyc1_srcs['dec'].values * skyc1_srcs['weight_ns'].values
+ skyc1_srcs["interim_ns"] = (
+ skyc1_srcs["dec"].values * skyc1_srcs["weight_ns"].values
)
- return skyc1_srcs.drop(['ra_source', 'dec_source'], axis=1)
+ return skyc1_srcs.drop(["ra_source", "dec_source"], axis=1)
skyc1 = SkyCoord(
- skyc1_srcs['ra'].values,
- skyc1_srcs['dec'].values,
- unit=(u.deg, u.deg)
+ skyc1_srcs["ra"].values, skyc1_srcs["dec"].values, unit=(u.deg, u.deg)
)
for it, epoch in enumerate(unique_epochs[start_epoch:]):
- logger.info('Association iteration: #%i%s', it + 1, skyreg_tag)
+ logger.info("Association iteration: #%i%s", it + 1, skyreg_tag)
# load skyc2 source measurements and create SkyCoord
- images = (
- images_df.loc[images_df['epoch'] == epoch, 'image_dj'].to_list()
- )
+ images = images_df.loc[images_df["epoch"] == epoch, "image_dj"].to_list()
max_beam_maj = (
- images_df.loc[images_df['epoch'] == epoch, 'image_dj']
+ images_df.loc[images_df["epoch"] == epoch, "image_dj"]
.apply(lambda x: x.beam_bmaj)
.max()
)
skyc2_srcs = prep_skysrc_df(
- images,
- config["measurements"]["flux_fractional_error"],
- duplicate_limit
+ images, config["measurements"]["flux_fractional_error"], duplicate_limit
)
- skyc2_srcs['epoch'] = epoch
+ skyc2_srcs["epoch"] = epoch
skyc2 = SkyCoord(
- skyc2_srcs['ra'].values,
- skyc2_srcs['dec'].values,
- unit=(u.deg, u.deg)
+ skyc2_srcs["ra"].values, skyc2_srcs["dec"].values, unit=(u.deg, u.deg)
)
- if method == 'basic':
+ if method == "basic":
sources_df, skyc1_srcs = basic_association(
sources_df,
skyc1_srcs,
@@ -1254,14 +1118,11 @@ def association(
skyc2_srcs,
skyc2,
limit,
- id_incr_par_assoc
)
- elif method in ['advanced', 'deruiter']:
- if method == 'deruiter':
- bw_max = Angle(
- bw_limit * (max_beam_maj * 3600. / 2.) * u.arcsec
- )
+ elif method in ["advanced", "deruiter"]:
+ if method == "deruiter":
+ bw_max = Angle(bw_limit * (max_beam_maj * 3600.0 / 2.0) * u.arcsec)
else:
bw_max = limit
sources_df, skyc1_srcs = advanced_association(
@@ -1273,250 +1134,133 @@ def association(
skyc2,
dr_limit,
bw_max,
- id_incr_par_assoc
)
else:
- raise Exception('association method not implemented!')
+ raise Exception("association method not implemented!")
logger.info(
- 'Calculating weighted average RA and Dec for sources%s...',
- skyreg_tag
+ "Calculating weighted average RA and Dec for sources%s...", skyreg_tag
)
# account for RA wrapping
ra_wrap_mask = sources_df.ra <= 0.1
- sources_df['ra_wrap'] = sources_df.ra.values
- sources_df.loc[
- ra_wrap_mask, 'ra_wrap'
- ] = sources_df[ra_wrap_mask].ra.values + 360.
+ sources_df["ra_wrap"] = sources_df.ra.values
+ sources_df.loc[ra_wrap_mask, "ra_wrap"] = (
+ sources_df[ra_wrap_mask].ra.values + 360.0
+ )
- sources_df['interim_ew'] = (
- sources_df['ra_wrap'].values * sources_df['weight_ew'].values
+ sources_df["interim_ew"] = (
+ sources_df["ra_wrap"].values * sources_df["weight_ew"].values
)
- sources_df['interim_ns'] = (
- sources_df['dec'].values * sources_df['weight_ns'].values
+ sources_df["interim_ns"] = (
+ sources_df["dec"].values * sources_df["weight_ns"].values
)
- sources_df = sources_df.drop(['ra_wrap'], axis=1)
+ sources_df = sources_df.drop(["ra_wrap"], axis=1)
- tmp_srcs_df = (
- sources_df.loc[
- (sources_df['source'] != -1) & (sources_df['forced'] == False),
- [
- 'ra', 'dec', 'uncertainty_ew', 'uncertainty_ns',
- 'source', 'interim_ew', 'interim_ns', 'weight_ew',
- 'weight_ns'
- ]
- ]
- .groupby('source')
- )
+ tmp_srcs_df = sources_df.loc[
+ (sources_df["source"].notnull()) & (sources_df["forced"] == False),
+ [
+ "ra",
+ "dec",
+ "uncertainty_ew",
+ "uncertainty_ns",
+ "source",
+ "interim_ew",
+ "interim_ns",
+ "weight_ew",
+ "weight_ns",
+ ],
+ ].groupby("source")
stats = StopWatch()
- wm_ra = tmp_srcs_df['interim_ew'].sum() / tmp_srcs_df['weight_ew'].sum()
- wm_uncertainty_ew = 1. / np.sqrt(tmp_srcs_df['weight_ew'].sum())
+ wm_ra = tmp_srcs_df["interim_ew"].sum() / tmp_srcs_df["weight_ew"].sum()
+ wm_uncertainty_ew = 1.0 / np.sqrt(tmp_srcs_df["weight_ew"].sum())
- wm_dec = tmp_srcs_df['interim_ns'].sum() / tmp_srcs_df['weight_ns'].sum()
- wm_uncertainty_ns = 1. / np.sqrt(tmp_srcs_df['weight_ns'].sum())
+ wm_dec = tmp_srcs_df["interim_ns"].sum() / tmp_srcs_df["weight_ns"].sum()
+ wm_uncertainty_ns = 1.0 / np.sqrt(tmp_srcs_df["weight_ns"].sum())
weighted_df = (
pd.concat(
[wm_ra, wm_uncertainty_ew, wm_dec, wm_uncertainty_ns],
axis=1,
- sort=False
+ sort=False,
)
.reset_index()
.rename(
columns={
- 0: 'ra',
- 'weight_ew': 'uncertainty_ew',
- 1: 'dec',
- 'weight_ns': 'uncertainty_ns'
- })
+ 0: "ra",
+ "weight_ew": "uncertainty_ew",
+ 1: "dec",
+ "weight_ns": "uncertainty_ns",
+ }
+ )
)
# correct the RA wrapping
- ra_wrap_mask = weighted_df.ra >= 360.
- weighted_df.loc[
- ra_wrap_mask, 'ra'
- ] = weighted_df[ra_wrap_mask].ra.values - 360.
+ ra_wrap_mask = weighted_df.ra >= 360.0
+ weighted_df.loc[ra_wrap_mask, "ra"] = (
+ weighted_df[ra_wrap_mask].ra.values - 360.0
+ )
- logger.debug('Groupby concat time %f', stats.reset())
+ logger.debug("Groupby concat time %f", stats.reset())
logger.info(
- 'Finalising base sources catalogue ready for next iteration%s...',
- skyreg_tag
+ "Finalising base sources catalogue ready for next iteration%s...",
+ skyreg_tag,
)
# merge the weighted ra and dec and replace the values
skyc1_srcs = skyc1_srcs.merge(
- weighted_df,
- on='source',
- how='left',
- suffixes=('', '_skyc2')
+ weighted_df, on="source", how="left", suffixes=("", "_skyc2")
)
del tmp_srcs_df, weighted_df
- skyc1_srcs['ra'] = skyc1_srcs['ra_skyc2']
- skyc1_srcs['dec'] = skyc1_srcs['dec_skyc2']
- skyc1_srcs['uncertainty_ew'] = skyc1_srcs['uncertainty_ew_skyc2']
- skyc1_srcs['uncertainty_ns'] = skyc1_srcs['uncertainty_ns_skyc2']
+ skyc1_srcs["ra"] = skyc1_srcs["ra_skyc2"]
+ skyc1_srcs["dec"] = skyc1_srcs["dec_skyc2"]
+ skyc1_srcs["uncertainty_ew"] = skyc1_srcs["uncertainty_ew_skyc2"]
+ skyc1_srcs["uncertainty_ns"] = skyc1_srcs["uncertainty_ns_skyc2"]
skyc1_srcs = skyc1_srcs.drop(
- [
- 'ra_skyc2',
- 'dec_skyc2',
- 'uncertainty_ew_skyc2',
- 'uncertainty_ns_skyc2'
- ], axis=1
+ ["ra_skyc2", "dec_skyc2", "uncertainty_ew_skyc2", "uncertainty_ns_skyc2"],
+ axis=1,
)
# generate new sky coord ready for next iteration
skyc1 = SkyCoord(
- skyc1_srcs['ra'].values,
- skyc1_srcs['dec'].values,
- unit=(u.deg, u.deg)
+ skyc1_srcs["ra"].values, skyc1_srcs["dec"].values, unit=(u.deg, u.deg)
)
# and update relations in skyc1
- skyc1_srcs = skyc1_srcs.drop('related', axis=1)
+ skyc1_srcs = skyc1_srcs.drop("related", axis=1)
relations_unique = pd.DataFrame(
- sources_df[sources_df['related'].notna()]
- .explode('related')
- .groupby('source')['related']
+ sources_df[sources_df["related"].notna()]
+ .explode("related")
+ .groupby("source")["related"]
.apply(lambda x: x.unique().tolist())
)
skyc1_srcs = skyc1_srcs.merge(
- relations_unique, how='left', left_on='source', right_index=True
+ relations_unique, how="left", left_on="source", right_index=True
)
- logger.info(
- 'Association iteration #%i complete%s.', it + 1, skyreg_tag
- )
+ logger.info("Association iteration #%i complete%s.", it + 1, skyreg_tag)
# End of iteration over images, ra and dec columns are actually the
# average over each iteration so remove ave ra and ave dec used for
# calculation and use ra_source and dec_source columns
- sources_df = (
- sources_df.drop(['ra', 'dec'], axis=1)
- .rename(columns={'ra_source': 'ra', 'dec_source': 'dec'})
+ sources_df = sources_df.drop(["ra", "dec"], axis=1).rename(
+ columns={"ra_source": "ra", "dec_source": "dec"}
)
del skyc1_srcs, skyc2_srcs
logger.info(
- 'Total association time: %.2f seconds%s.',
- timer.reset_init(),
- skyreg_tag
+ "Total association time: %.2f seconds%s.", timer.reset_init(), skyreg_tag
)
return sources_df
-def _correct_parallel_source_ids(
- df: pd.DataFrame, correction: int) -> pd.DataFrame:
- """
- This function is to correct the source ids after the combination of
- the associaiton dataframes produced by parallel association - as source
- ids will be duplicated if left.
-
- Args:
- df:
- Holds the measurements associated into sources. The output of
- of the association step (sources_df).
- correction:
- The value to add to the source ids.
-
- Returns:
- The input df with corrected source ids and relations.
- """
- df['source'] = df['source'].values + correction
- related_mask = df['related'].notna()
-
- new_relations = df.loc[
- related_mask, 'related'
- ].explode() + correction
-
- df.loc[
- df[related_mask].index.values, 'related'
- ] = new_relations.groupby(level=0).apply(
- lambda x: x.values.tolist()
- )
-
- return df
-
-
-def _correct_parallel_source_ids_add_mode(
- df: pd.DataFrame, done_source_ids: List[int],
- start_elem: int) -> Tuple[pd.DataFrame, int]:
- """
- This function is to correct the source ids after the combination of
- the associaiton dataframes produced by parallel association - as source
- ids will be duplicated if left - specifically for add mode.
-
- When add mode is being used the 'old' sources require the ID to remain
- the same with only the new ones being changed. The next start elem also
- needs to be dynamically updated with every skyreg_group loop.
-
- Args:
- df:
- Holds the measurements associated into sources. The output of of
- the association step (sources_df).
- done_source_ids:
- A list of the 'old' source ids that need to remain the same.
- start_elem:
- The start elem number for the new source ids.
-
- Returns:
- The input dataframe with corrected source ids and relations.
- The new start elem for the next group.
- """
- # When using add_mode the correction becomes easier with the increment
- # as there's a clear difference between old and new.
- # old ones do not need to be corrected
-
- # get a mask of those that need to be corrected
- to_correct_mask = ~df['source'].isin(done_source_ids)
-
- # check that there are any to correct
- if not np.any(to_correct_mask):
- # there are no ids to correct we can just return the input
- # next start elem is just the same as the input as well
- return df[['source', 'related']], start_elem
-
- # create a new column for the new id
- df['new_source'] = df['source']
- # how many unique new sources
- to_correct_source_ids = df.loc[to_correct_mask, 'source'].unique()
- # create the range of new ids
- new_ids = list(
- range(start_elem, start_elem + to_correct_source_ids.shape[0]))
- # create a map of old source to new source
- source_id_map = dict(zip(to_correct_source_ids, new_ids))
- # get and apply the new ids to the new column
- df.loc[to_correct_mask, 'new_source'] = (
- df.loc[to_correct_mask, 'new_source'].map(source_id_map)
- )
- # regenrate the map
- source_id_map = dict(zip(df.source.values, df.new_source.values))
- # get mask of non-nan relations
- related_mask = df['related'].notna()
- # get the relations
- new_relations = df.loc[related_mask, 'related'].explode()
- # map the new values
- new_relations = new_relations.map(source_id_map)
- # group them back and form lists again
- new_relations = new_relations.groupby(level=0).apply(
- lambda x: x.values.tolist())
- # apply corrected relations to results
- df.loc[df[related_mask].index.values, 'related'] = new_relations
- # drop the old sources and replace
- df = df.drop('source', axis=1).rename(columns={'new_source': 'source'})
- # define what the next start elem will be
- next_start_elem = new_ids[-1] + 1
-
- return df[['source', 'related']], next_start_elem
-
-
def parallel_association(
images_df: pd.DataFrame,
limit: Angle,
@@ -1528,7 +1272,7 @@ def parallel_association(
add_mode: bool,
previous_parquets: Dict[str, str],
done_images_df: pd.DataFrame,
- done_source_ids: List[int]
+ done_source_ids: List[int],
) -> pd.DataFrame:
"""
Launches association on different sky region groups in parallel using Dask.
@@ -1548,51 +1292,46 @@ def parallel_association(
association with corrected source ids.
"""
logger.info(
- "Running parallel association for %i sky region groups.",
- n_skyregion_groups
+ "Running parallel association for %i sky region groups.", n_skyregion_groups
)
timer = StopWatch()
meta = {
- 'id': 'i',
- 'uncertainty_ew': 'f',
- 'weight_ew': 'f',
- 'uncertainty_ns': 'f',
- 'weight_ns': 'f',
- 'flux_int': 'f',
- 'flux_int_err': 'f',
- 'flux_int_isl_ratio': 'f',
- 'flux_peak': 'f',
- 'flux_peak_err': 'f',
- 'flux_peak_isl_ratio': 'f',
- 'forced': '?',
- 'compactness': 'f',
- 'has_siblings': '?',
- 'snr': 'f',
- 'image': 'U',
- 'datetime': 'datetime64[ns]',
- 'source': 'i',
- 'ra': 'f',
- 'dec': 'f',
- 'd2d': 'f',
- 'dr': 'f',
- 'related': 'O',
- 'epoch': 'i',
- 'interim_ew': 'f',
- 'interim_ns': 'f',
+ "id": "U36",
+ "uncertainty_ew": "f",
+ "weight_ew": "f",
+ "uncertainty_ns": "f",
+ "weight_ns": "f",
+ "flux_int": "f",
+ "flux_int_err": "f",
+ "flux_int_isl_ratio": "f",
+ "flux_peak": "f",
+ "flux_peak_err": "f",
+ "flux_peak_isl_ratio": "f",
+ "forced": "?",
+ "compactness": "f",
+ "has_siblings": "?",
+ "snr": "f",
+ "image": "U36",
+ "datetime": "datetime64[ns]",
+ "source": "U36",
+ "ra": "f",
+ "dec": "f",
+ "d2d": "f",
+ "dr": "f",
+ "related": "O",
+ "epoch": "i",
+ "interim_ew": "f",
+ "interim_ns": "f",
}
- # Add an increment to any new source values when using add_mode to avoid
- # getting duplicates in the result laater
- id_incr_par_assoc = max(done_source_ids) if add_mode else 0
-
n_cpu = cpu_count() - 1
# pass each skyreg_group through the normal association process.
results = (
dd.from_pandas(images_df, n_cpu)
- .groupby('skyreg_group')
+ .groupby("skyreg_group")
.apply(
association,
limit=limit,
@@ -1603,10 +1342,10 @@ def parallel_association(
add_mode=add_mode,
previous_parquets=previous_parquets,
done_images_df=done_images_df,
- id_incr_par_assoc=id_incr_par_assoc,
parallel=True,
- meta=meta
- ).compute(n_workers=n_cpu, scheduler='processes')
+ meta=meta,
+ )
+ .compute(n_workers=n_cpu, scheduler="processes")
)
# results are the normal dataframe of results with the columns:
@@ -1618,11 +1357,10 @@ def parallel_association(
# The index however is now a multi index with the skyregion group and
# a general result index. Hence the general result index is repeated for
- # each skyreg_group along with the source_ids. This needs to be collapsed
- # and the source id's corrected.
+ # each skyreg_group along with the source_ids. This needs to be collapsed.
# Index example:
- # id
+ # id (now UUIDs)
# skyreg_group
# --------------------------
# 2 0 15640
@@ -1637,51 +1375,9 @@ def parallel_association(
# 46978 54161
# 46979 54164
- # Get the indexes (skyreg_groups) to loop over for source id correction
- indexes = results.index.levels[0].values
-
- if add_mode:
- # Need to correct all skyreg_groups.
- # First get the starting id for new sources.
- new_id = max(done_source_ids) + 1
- for i in indexes:
- corr_df, new_id = _correct_parallel_source_ids_add_mode(
- results.loc[i, ['source', 'related']],
- done_source_ids,
- new_id
- )
- results.loc[
- (i, slice(None)), ['source', 'related']
- ] = corr_df.values
- else:
- # The first index acts as the base, so the others are looped over and
- # corrected.
- for i, val in enumerate(indexes):
- # skip first one, makes the enumerate easier to deal with
- if i == 0:
- continue
- # Get the maximum source ID from the previous group.
- max_id = results.loc[indexes[i - 1]].source.max()
- # Run through the correction function, only the 'source' and
- # 'related'
- # columns are passed and returned (corrected).
- corr_df = _correct_parallel_source_ids(
- results.loc[val, ['source', 'related']],
- max_id
- )
- # replace the values in the results with the corrected source and
- # related values
- results.loc[
- (val, slice(None)), ['source', 'related']
- ] = corr_df.values
-
- del corr_df
-
# reset the indeex of the final corrected and collapsed result
results = results.reset_index(drop=True)
- logger.info(
- 'Total parallel association time: %.2f seconds', timer.reset_init()
- )
+ logger.info("Total parallel association time: %.2f seconds", timer.reset_init())
return results
diff --git a/vast_pipeline/pipeline/finalise.py b/vast_pipeline/pipeline/finalise.py
index b02a27d49..ed23c9694 100644
--- a/vast_pipeline/pipeline/finalise.py
+++ b/vast_pipeline/pipeline/finalise.py
@@ -10,8 +10,10 @@
from vast_pipeline.models import Run
from vast_pipeline.utils.utils import StopWatch, optimize_floats, optimize_ints
from vast_pipeline.pipeline.loading import (
- make_upload_associations, make_upload_sources, make_upload_related_sources,
- update_sources
+ make_upload_associations,
+ make_upload_sources,
+ make_upload_related_sources,
+ update_sources,
)
from vast_pipeline.pipeline.pairs import calculate_measurement_pair_metrics
from vast_pipeline.pipeline.utils import parallel_groupby
@@ -64,17 +66,27 @@ def calculate_measurement_pair_aggregate_metrics(
)
else:
pair_agg_metrics = measurement_pairs_df.iloc[
- check_df
- .groupby("source")
- .agg(m_abs_max_idx=(f"m_{flux_type}", lambda x: x.abs().idxmax()),)
- .astype(np.int32)["m_abs_max_idx"] # cast row indices to int and select them
+ check_df.groupby("source")
+ .agg(
+ m_abs_max_idx=(f"m_{flux_type}", lambda x: x.abs().idxmax()),
+ )
+ .astype(np.int32)[
+ "m_abs_max_idx"
+ ] # cast row indices to int and select them
.reset_index(drop=True) # keep only the row indices
][[f"vs_{flux_type}", f"m_{flux_type}", "source"]]
- pair_agg_metrics = pair_agg_metrics.abs().rename(columns={
- f"vs_{flux_type}": f"vs_abs_significant_max_{flux_type}",
- f"m_{flux_type}": f"m_abs_significant_max_{flux_type}",
- }).set_index('source')
+ pair_agg_metrics = (
+ pair_agg_metrics.set_index("source")
+ .abs()
+ .rename(
+ columns={
+ f"vs_{flux_type}": f"vs_abs_significant_max_{flux_type}",
+ f"m_{flux_type}": f"m_abs_significant_max_{flux_type}",
+ }
+ )
+ )
+
return pair_agg_metrics
@@ -86,7 +98,7 @@ def final_operations(
source_aggregate_pair_metrics_min_abs_vs: float,
add_mode: bool,
done_source_ids: List[int],
- previous_parquets: Dict[str, str]
+ previous_parquets: Dict[str, str],
) -> Tuple[int, int]:
"""
Performs the final operations of the pipeline:
@@ -129,11 +141,10 @@ def final_operations(
# calculate source fields
logger.info(
- 'Calculating statistics for %i sources...',
- sources_df.source.unique().shape[0]
+ "Calculating statistics for %i sources...", sources_df.source.unique().shape[0]
)
srcs_df = parallel_groupby(sources_df)
- logger.info('Groupby-apply time: %.2f seconds', timer.reset())
+ logger.info("Groupby-apply time: %.2f seconds", timer.reset())
# add new sources
srcs_df["new"] = srcs_df.index.isin(new_sources_df.index)
@@ -148,32 +159,31 @@ def final_operations(
# calculate nearest neighbour
srcs_skycoord = SkyCoord(
- srcs_df['wavg_ra'].values,
- srcs_df['wavg_dec'].values,
- unit=(u.deg, u.deg)
- )
- idx, d2d, _ = srcs_skycoord.match_to_catalog_sky(
- srcs_skycoord,
- nthneighbor=2
+ srcs_df["wavg_ra"].values, srcs_df["wavg_dec"].values, unit=(u.deg, u.deg)
)
+ _, d2d, _ = srcs_skycoord.match_to_catalog_sky(srcs_skycoord, nthneighbor=2)
# add the separation distance in degrees
- srcs_df['n_neighbour_dist'] = d2d.deg
+ srcs_df["n_neighbour_dist"] = d2d.deg
# create measurement pairs, aka 2-epoch metrics
if calculate_pairs:
timer.reset()
measurement_pairs_df = calculate_measurement_pair_metrics(sources_df)
- logger.info('Measurement pair metrics time: %.2f seconds', timer.reset())
+ logger.info("Measurement pair metrics time: %.2f seconds", timer.reset())
# calculate measurement pair metric aggregates for sources by finding the row indices
# of the aggregate max of the abs(m) metric for each flux type.
pair_agg_metrics = pd.merge(
calculate_measurement_pair_aggregate_metrics(
- measurement_pairs_df, source_aggregate_pair_metrics_min_abs_vs, flux_type="peak",
+ measurement_pairs_df,
+ source_aggregate_pair_metrics_min_abs_vs,
+ flux_type="peak",
),
calculate_measurement_pair_aggregate_metrics(
- measurement_pairs_df, source_aggregate_pair_metrics_min_abs_vs, flux_type="int",
+ measurement_pairs_df,
+ source_aggregate_pair_metrics_min_abs_vs,
+ flux_type="int",
),
how="outer",
left_index=True,
@@ -182,13 +192,17 @@ def final_operations(
# join with sources and replace agg metrics NaNs with 0 as the DataTables API JSON
# serialization doesn't like them
- srcs_df = srcs_df.join(pair_agg_metrics).fillna(value={
- "vs_abs_significant_max_peak": 0.0,
- "m_abs_significant_max_peak": 0.0,
- "vs_abs_significant_max_int": 0.0,
- "m_abs_significant_max_int": 0.0,
- })
- logger.info("Measurement pair aggregate metrics time: %.2f seconds", timer.reset())
+ srcs_df = srcs_df.join(pair_agg_metrics).fillna(
+ value={
+ "vs_abs_significant_max_peak": 0.0,
+ "m_abs_significant_max_peak": 0.0,
+ "vs_abs_significant_max_int": 0.0,
+ "m_abs_significant_max_int": 0.0,
+ }
+ )
+ logger.info(
+ "Measurement pair aggregate metrics time: %.2f seconds", timer.reset()
+ )
else:
logger.info(
"Skipping measurement pair metric calculation as specified in the run configuration."
@@ -198,118 +212,109 @@ def final_operations(
if add_mode:
# if add mode is being used some sources need to updated where as some
# need to be newly uploaded.
- # upload new ones first (new id's are fetched)
+ # upload new ones first
src_done_mask = srcs_df.index.isin(done_source_ids)
srcs_df_upload = srcs_df.loc[~src_done_mask].copy()
- srcs_df_upload = make_upload_sources(srcs_df_upload, p_run, add_mode)
+ make_upload_sources(srcs_df_upload, p_run, add_mode)
# And now update
srcs_df_update = srcs_df.loc[src_done_mask].copy()
- logger.info(
- f"Updating {srcs_df_update.shape[0]} sources with new metrics.")
+ logger.info(f"Updating {srcs_df_update.shape[0]} sources with new metrics.")
srcs_df = update_sources(srcs_df_update, batch_size=1000)
# Add back together
if not srcs_df_upload.empty:
srcs_df = pd.concat([srcs_df, srcs_df_upload])
else:
- srcs_df = make_upload_sources(srcs_df, p_run, add_mode)
+ make_upload_sources(srcs_df, p_run, add_mode)
# gather the related df, upload to db and save to parquet file
# the df will look like
#
# from_source_id to_source_id
- # source
- # 714 60 14396
- # 1211 94 12961
- #
- # the index ('source') has the initial id generated by the pipeline to
- # identify unique sources, the 'from_source_id' column has the django
- # model id (in db), the 'to_source_id' has the pipeline index
+ # index
+ # 0 60 14396
+ # 1 94 12961
related_df = (
- srcs_df.loc[srcs_df["related_list"] != -1, ["id", "related_list"]]
+ srcs_df.loc[srcs_df["related_list"] != -1, ["related_list"]]
.explode("related_list")
- .rename(columns={"id": "from_source_id", "related_list": "to_source_id"})
+ .reset_index()
+ .rename(columns={"source": "from_source_id", "related_list": "to_source_id"})
)
- # for the column 'from_source_id', replace relation source ids with db id
- related_df["to_source_id"] = related_df["to_source_id"].map(srcs_df["id"].to_dict())
# drop relationships with the same source
- related_df = related_df[related_df["from_source_id"] != related_df["to_source_id"]]
+ related_df = related_df.loc[
+ related_df["from_source_id"] != related_df["to_source_id"]
+ ]
# write symmetrical relations to parquet
- related_df.to_parquet(
- os.path.join(p_run.path, 'relations.parquet'),
- index=False
- )
+ related_df.to_parquet(os.path.join(p_run.path, "relations.parquet"), index=False)
# upload the relations to DB
# check for add_mode first
if add_mode:
# Load old relations so the already uploaded ones can be removed
- old_relations = (
- pd.read_parquet(previous_parquets['relations'])
- )
+ old_relations = pd.read_parquet(previous_parquets["relations"])
- related_df = (
- pd.concat([related_df, old_relations], ignore_index=True)
- .drop_duplicates(keep=False)
- )
- logger.debug(f'Add mode: #{related_df.shape[0]} relations to upload.')
+ related_df = pd.concat(
+ [related_df, old_relations], ignore_index=True
+ ).drop_duplicates(keep=False)
+ logger.debug(f"Add mode: #{related_df.shape[0]} relations to upload.")
make_upload_related_sources(related_df)
del related_df
# write sources to parquet file
- srcs_df = srcs_df.drop(["related_list", "img_list"], axis=1)
+ cols_to_drop = ["related_list", "img_list"]
+
+ if add_mode:
+ cols_to_drop.append("id")
+
+ srcs_df = srcs_df.drop(cols_to_drop, axis=1)
+
(
- srcs_df.set_index('id') # set the index to db ids, dropping the source idx
- .to_parquet(os.path.join(p_run.path, 'sources.parquet'))
+ srcs_df.to_parquet( # set the index to db ids, dropping the source idx
+ os.path.join(p_run.path, "sources.parquet")
+ )
)
- # update measurments with sources to get associations
- sources_df = (
- sources_df.drop('related', axis=1)
- .merge(srcs_df.rename(columns={'id': 'source_id'}), on='source')
- )
+ # update measurements with sources to get associations
+ sources_df = sources_df.drop("related", axis=1)
if add_mode:
# Load old associations so the already uploaded ones can be removed
- old_assoications = (
- pd.read_parquet(previous_parquets['associations'])
- .rename(columns={'meas_id': 'id'})
+ old_associations = pd.read_parquet(previous_parquets["associations"]).rename(
+ columns={"meas_id": "id", "source_id": "source"}
)
sources_df_upload = pd.concat(
- [sources_df, old_assoications],
+ [sources_df, old_associations],
ignore_index=True
)
sources_df_upload = sources_df_upload.drop_duplicates(
- ['source_id', 'id', 'd2d', 'dr'], keep=False
+ ["source", "id", "d2d", "dr"], keep=False
)
- logger.debug(
- f'Add mode: #{sources_df_upload.shape[0]} associations to upload.')
+ logger.debug(f"Add mode: #{sources_df_upload.shape[0]} associations to upload.")
else:
sources_df_upload = sources_df
# upload associations into DB
- make_upload_associations(sources_df_upload)
+ make_upload_associations(sources_df_upload[["id", "source", "d2d", "dr"]])
# write associations to parquet file
- sources_df.rename(columns={'id': 'meas_id'})[
- ['source_id', 'meas_id', 'd2d', 'dr']
- ].to_parquet(os.path.join(p_run.path, 'associations.parquet'))
+ sources_df.rename(columns={"id": "meas_id", "source": "source_id"})[
+ ["source_id", "meas_id", "d2d", "dr"]
+ ].to_parquet(os.path.join(p_run.path, "associations.parquet"))
if calculate_pairs:
- # get the Source object primary keys for the measurement pairs
- measurement_pairs_df = measurement_pairs_df.join(
- srcs_df.id.rename("source_id"), on="source"
- )
-
# optimize measurement pair DataFrame and save to parquet file
measurement_pairs_df = optimize_ints(
optimize_floats(
- measurement_pairs_df.drop(columns=["source"]).rename(
- columns={"id_a": "meas_id_a", "id_b": "meas_id_b"}
+ measurement_pairs_df.rename(
+ columns={
+ "id_a": "meas_id_a",
+ "id_b": "meas_id_b",
+ "source": "source_id",
+ }
)
)
)
@@ -319,8 +324,8 @@ def final_operations(
logger.info("Total final operations time: %.2f seconds", timer.reset_init())
- nr_sources = srcs_df["id"].count()
- nr_new_sources = srcs_df['new'].sum()
+ nr_sources = srcs_df.shape[0]
+ nr_new_sources = srcs_df["new"].sum()
# calculate and return total number of extracted sources
return (nr_sources, nr_new_sources)
diff --git a/vast_pipeline/pipeline/forced_extraction.py b/vast_pipeline/pipeline/forced_extraction.py
index eef7a3ed8..112710b9f 100644
--- a/vast_pipeline/pipeline/forced_extraction.py
+++ b/vast_pipeline/pipeline/forced_extraction.py
@@ -1,6 +1,7 @@
import os
import logging
import datetime
+import uuid
import numpy as np
import pandas as pd
import dask.dataframe as dd
@@ -27,7 +28,7 @@
def remove_forced_meas(run_path: str) -> None:
- '''
+ """
Remove forced measurements from the database if forced parquet files
are found.
@@ -37,34 +38,29 @@ def remove_forced_meas(run_path: str) -> None:
Returns:
None
- '''
- path_glob = glob(
- os.path.join(run_path, 'forced_measurements_*.parquet')
- )
+ """
+ path_glob = glob(os.path.join(run_path, "forced_measurements_*.parquet"))
if path_glob:
- ids = (
- dd.read_parquet(path_glob, columns='id')
- .values
- .compute()
- .tolist()
- )
+ ids = dd.read_parquet(path_glob, columns="id").values.compute().tolist()
obj_to_delete = Measurement.objects.filter(id__in=ids)
del ids
if obj_to_delete.exists():
with transaction.atomic():
n_del, detail_del = obj_to_delete.delete()
logger.info(
- ('Deleting all previous forced measurement and association'
- ' objects for this run. Total objects deleted: %i'),
+ (
+ "Deleting all previous forced measurement and association"
+ " objects for this run. Total objects deleted: %i"
+ ),
n_del,
)
- logger.debug('(type, #deleted): %s', detail_del)
+ logger.debug("(type, #deleted): %s", detail_del)
def get_data_from_parquet(
file_and_image_id: Tuple[str, int], p_run_path: str, add_mode: bool = False
) -> Dict:
- '''
+ """
Get the prefix, max id and image id from the measurements parquets
Args:
@@ -80,30 +76,27 @@ def get_data_from_parquet(
Returns:
Dictionary with prefix string, an interger max_id and a string with the
id of the image.
- '''
+ """
file, image_id = file_and_image_id
if add_mode:
image_name = file.split("/")[-2]
forced_parquet = os.path.join(
- p_run_path,
- f"forced_measurements_{image_name}.parquet"
+ p_run_path, f"forced_measurements_{image_name}.parquet"
)
if os.path.isfile(forced_parquet):
file = forced_parquet
# get max component id from parquet file
- df = pd.read_parquet(file, columns=['island_id', 'image_id'])
+ df = pd.read_parquet(file, columns=["island_id", "image_id"])
if len(df) > 0:
- prefix = df['island_id'].iloc[0].rsplit('_', maxsplit=1)[0] + '_'
+ prefix = df["island_id"].iloc[0].rsplit("_", maxsplit=1)[0] + "_"
max_id = (
- df['island_id'].str.rsplit('_', n=1)
- .str.get(-1)
- .astype(int)
- .values.max() + 1
+ df["island_id"].str.rsplit("_", n=1).str.get(-1).astype(int).values.max()
+ + 1
)
else:
prefix = "island_"
max_id = 1
- return {'prefix': prefix, 'max_id': max_id, 'id': image_id}
+ return {"prefix": prefix, "max_id": max_id, "id": image_id}
def _forcedphot_preload(image: str,
@@ -169,9 +162,7 @@ def extract_from_image(
# create the skycoord obj to pass to the forced extraction
# see usage https://github.com/dlakaplan/forced_phot
P_islands = SkyCoord(
- df['wavg_ra'].values,
- df['wavg_dec'].values,
- unit=(u.deg, u.deg)
+ df["wavg_ra"].values, df["wavg_dec"].values, unit=(u.deg, u.deg)
)
# load the image, background and noisemaps into memory
# a dedicated function may seem unneccesary, but will be useful if we
@@ -187,19 +178,25 @@ def extract_from_image(
P_islands,
cluster_threshold=cluster_threshold,
allow_nan=allow_nan,
- edge_buffer=edge_buffer
+ edge_buffer=edge_buffer,
)
- df['flux_int'] = flux * 1.e3
- df['flux_int_err'] = flux_err * 1.e3
- df['chi_squared_fit'] = chisq
+ df["flux_int"] = flux * 1.0e3
+ df["flux_int_err"] = flux_err * 1.0e3
+ df["chi_squared_fit"] = chisq
- return {'df': df, 'image': df['image_name'].iloc[0]}
+ return {"df": df, "image": df["image_name"].iloc[0]}
def finalise_forced_dfs(
- df: pd.DataFrame, prefix: str, max_id: int, beam_bmaj: float,
- beam_bmin: float, beam_bpa: float, id: int, datetime: datetime.datetime,
- image: str
+ df: pd.DataFrame,
+ prefix: str,
+ max_id: int,
+ beam_bmaj: float,
+ beam_bmin: float,
+ beam_bpa: float,
+ id: int,
+ datetime: datetime.datetime,
+ image: str,
) -> pd.DataFrame:
"""
Compute populate leftover columns for the dataframe with forced
@@ -231,32 +228,35 @@ def finalise_forced_dfs(
name, bmaj, bmin, pa, image_id, time.
"""
# make up the measurements name from the image island_id and component_id
- df['island_id'] = np.char.add(
- prefix,
- np.arange(max_id, max_id + df.shape[0]).astype(str)
+ df["island_id"] = np.char.add(
+ prefix, np.arange(max_id, max_id + df.shape[0]).astype(str)
)
- df['component_id'] = df['island_id'].str.replace(
- 'island', 'component'
- ) + 'a'
- img_prefix = image.split('.')[0] + '_'
- df['name'] = img_prefix + df['component_id']
+ df["component_id"] = df["island_id"].str.replace("island", "component") + "a"
+ img_prefix = image.split(".")[0] + "_"
+ df["name"] = img_prefix + df["component_id"]
# assign all the other columns
# convert fluxes to mJy
# store source bmaj and bmin in arcsec
- df['bmaj'] = beam_bmaj * 3600.
- df['bmin'] = beam_bmin * 3600.
- df['pa'] = beam_bpa
+ df["bmaj"] = beam_bmaj * 3600.0
+ df["bmin"] = beam_bmin * 3600.0
+ df["pa"] = beam_bpa
# add image id and time
- df['image_id'] = id
- df['time'] = datetime
+ df["image_id"] = id
+ df["time"] = datetime
return df
def parallel_extraction(
- df: pd.DataFrame, df_images: pd.DataFrame, df_sources: pd.DataFrame,
- min_sigma: float, edge_buffer: float, cluster_threshold: float,
- allow_nan: bool, add_mode: bool, p_run_path: str
+ df: pd.DataFrame,
+ df_images: pd.DataFrame,
+ df_sources: pd.DataFrame,
+ min_sigma: float,
+ edge_buffer: float,
+ cluster_threshold: float,
+ allow_nan: bool,
+ add_mode: bool,
+ p_run_path: str,
) -> pd.DataFrame:
"""
Parallelize forced extraction with Dask
@@ -294,33 +294,26 @@ def parallel_extraction(
"""
# explode the lists in 'img_diff' column (this will make a copy of the df)
out = (
- df.rename(columns={'img_diff': 'image', 'source': 'source_tmp_id'})
+ df.rename(columns={"img_diff": "image", "source": "source_tmp_id"})
# merge the rms_min column from df_images
- .merge(
- df_images[['rms_min']],
- left_on='image',
- right_on='name',
- how='left'
- )
- .rename(columns={'rms_min': 'image_rms_min'})
+ .merge(df_images[["rms_min"]], left_on="image", right_on="name", how="left")
+ .rename(columns={"rms_min": "image_rms_min"})
# merge the measurements columns 'source', 'image', 'flux_peak'
.merge(
df_sources,
- left_on=['source_tmp_id', 'detection'],
- right_on=['source', 'image'],
- how='left'
+ left_on=["source_tmp_id", "detection"],
+ right_on=["source", "image"],
+ how="left",
)
- .drop(columns=['image_y', 'source'])
- .rename(columns={'image_x': 'image'})
+ .drop(columns=["image_y", "source"])
+ .rename(columns={"image_x": "image"})
)
# drop the source for which we would have no hope of detecting
predrop_shape = out.shape[0]
- out['max_snr'] = out['flux_peak'].values / out['image_rms_min'].values
- out = out[out['max_snr'] > min_sigma].reset_index(drop=True)
- logger.debug("Min forced sigma dropped %i sources",
- predrop_shape - out.shape[0]
- )
+ out["max_snr"] = out["flux_peak"].values / out["image_rms_min"].values
+ out = out[out["max_snr"] > min_sigma].reset_index(drop=True)
+ logger.debug("Min forced sigma dropped %i sources", predrop_shape - out.shape[0])
# drop some columns that are no longer needed and the df should look like
# out
@@ -332,13 +325,12 @@ def parallel_extraction(
# | 3 | 1353 | 322.094 | -4.44977 | VAST_2118-06A... | 1.879 |
# | 4 | 1387 | 321.734 | -6.82934 | VAST_2118-06A... | 1.61 |
- out = (
- out.drop(['max_snr', 'image_rms_min', 'detection'], axis=1)
- .rename(columns={'image': 'image_name'})
+ out = out.drop(["max_snr", "image_rms_min", "detection"], axis=1).rename(
+ columns={"image": "image_name"}
)
# get the unique images to extract from
- unique_images_to_extract = out['image_name'].unique().tolist()
+ unique_images_to_extract = out["image_name"].unique().tolist()
# create a list of dictionaries with image file paths and dataframes
# with data related to each images
@@ -346,68 +338,66 @@ def image_data_func(image_name: str) -> Dict[str, Any]:
# `out` refers to the `out` declared in nearest enclosing scope
nonlocal out
return {
- 'image_id': df_images.at[image_name, 'id'],
- 'image': df_images.at[image_name, 'path'],
- 'background': df_images.at[image_name, 'background_path'],
- 'noise': df_images.at[image_name, 'noise_path'],
- 'df': out[out['image_name'] == image_name]
+ "image_id": df_images.at[image_name, "id"],
+ "image": df_images.at[image_name, "path"],
+ "background": df_images.at[image_name, "background_path"],
+ "noise": df_images.at[image_name, "noise_path"],
+ "df": out[out["image_name"] == image_name],
}
+
list_to_map = list(map(image_data_func, unique_images_to_extract))
# create a list of all the measurements parquet files to extract data from,
# such as prefix and max_id
- list_meas_parquets = list(map(
- lambda image_name: (
- df_images.at[image_name, 'measurements_path'],
- df_images.at[image_name, 'id'],
- ),
- unique_images_to_extract
- ))
+ list_meas_parquets = list(
+ map(
+ lambda image_name: (
+ df_images.at[image_name, "measurements_path"],
+ df_images.at[image_name, "id"],
+ ),
+ unique_images_to_extract,
+ )
+ )
del out, unique_images_to_extract, image_data_func
# get a map of the columns that have a fixed value
mapping = (
- db.from_sequence(
- list_meas_parquets,
- npartitions=len(list_meas_parquets)
- )
+ db.from_sequence(list_meas_parquets, npartitions=len(list_meas_parquets))
.map(get_data_from_parquet, p_run_path, add_mode)
.compute()
)
mapping = pd.DataFrame(mapping)
# remove not used columns from images_df and merge into mapping
- col_to_drop = list(filter(
- lambda x: ('path' in x) or ('skyreg' in x),
- df_images.columns.values.tolist()
- ))
+ col_to_drop = list(
+ filter(
+ lambda x: ("path" in x) or ("skyreg" in x),
+ df_images.columns.values.tolist(),
+ )
+ )
mapping = (
mapping.merge(
- df_images.drop(col_to_drop, axis=1).reset_index(),
- on='id',
- how='left'
+ df_images.drop(col_to_drop, axis=1).reset_index(), on="id", how="left"
)
- .drop('rms_min', axis=1)
- .set_index('name')
+ .drop("rms_min", axis=1)
+ .set_index("name")
)
del col_to_drop
n_cpu = cpu_count() - 1
bags = db.from_sequence(list_to_map, npartitions=len(list_to_map))
- forced_dfs = (
- bags.map(lambda x: extract_from_image(
+ forced_dfs = bags.map(
+ lambda x: extract_from_image(
edge_buffer=edge_buffer,
cluster_threshold=cluster_threshold,
allow_nan=allow_nan,
- **x
- ))
- .compute()
- )
+ **x,
+ )
+ ).compute(scheduler="processes", num_workers=n_cpu)
del bags
# create intermediates dfs combining the mapping data and the forced
# extracted data from the images
- intermediate_df = list(map(
- lambda x: {**(mapping.loc[x['image'], :].to_dict()), **x},
- forced_dfs
- ))
+ intermediate_df = list(
+ map(lambda x: {**(mapping.loc[x["image"], :].to_dict()), **x}, forced_dfs)
+ )
# compute the rest of the columns
intermediate_df = (
@@ -415,21 +405,17 @@ def image_data_func(image_name: str) -> Dict[str, Any]:
.map(lambda x: finalise_forced_dfs(**x))
.compute()
)
- df_out = (
- pd.concat(intermediate_df, axis=0, sort=False)
- .rename(
- columns={
- 'wavg_ra': 'ra', 'wavg_dec': 'dec', 'image_name': 'image'
- }
- )
+ df_out = pd.concat(intermediate_df, axis=0, sort=False).rename(
+ columns={"wavg_ra": "ra", "wavg_dec": "dec", "image_name": "image"}
)
+ df_out["id"] = df_out.apply(lambda _: str(uuid.uuid4()), axis=1)
+
return df_out
-def write_group_to_parquet(
- df: pd.DataFrame, fname: str, add_mode: bool) -> None:
- '''
+def write_group_to_parquet(df: pd.DataFrame, fname: str, add_mode: bool) -> None:
+ """
Write a dataframe correpondent to a single group/image
to a parquet file.
@@ -443,8 +429,8 @@ def write_group_to_parquet(
Returns:
None
- '''
- out_df = df.drop(['d2d', 'dr', 'source', 'image'], axis=1)
+ """
+ out_df = df.drop(["d2d", "dr", "source", "image"], axis=1)
if os.path.isfile(fname) and add_mode:
exist_df = pd.read_parquet(fname)
out_df = pd.concat([exist_df, out_df])
@@ -455,8 +441,9 @@ def write_group_to_parquet(
def parallel_write_parquet(
- df: pd.DataFrame, run_path: str, add_mode: bool = False) -> None:
- '''
+ df: pd.DataFrame, run_path: str, add_mode: bool = False
+) -> None:
+ """
Parallelize writing parquet files for forced measurements.
Args:
@@ -469,30 +456,35 @@ def parallel_write_parquet(
Returns:
None
- '''
- images = df['image'].unique().tolist()
-
- def get_fname(n): return os.path.join(
- run_path,
- 'forced_measurements_' + n.replace('.', '_') + '.parquet'
+ """
+ images = df["image"].unique().tolist()
+ get_fname = lambda n: os.path.join(
+ run_path, "forced_measurements_" + n.replace(".", "_") + ".parquet"
)
- dfs = list(map(lambda x: (df[df['image'] == x], get_fname(x)), images))
+ dfs = list(map(lambda x: (df[df["image"] == x], get_fname(x)), images))
n_cpu = cpu_count() - 1
# writing parquets using Dask bag
bags = db.from_sequence(dfs)
- bags = bags.starmap(
- lambda df, fname: write_group_to_parquet(df, fname, add_mode))
+ bags = bags.starmap(lambda df, fname: write_group_to_parquet(df, fname, add_mode))
bags.compute(num_workers=n_cpu)
pass
def forced_extraction(
- sources_df: pd.DataFrame, cfg_err_ra: float, cfg_err_dec: float,
- p_run: Run, extr_df: pd.DataFrame, min_sigma: float, edge_buffer: float,
- cluster_threshold: float, allow_nan: bool, add_mode: bool,
- done_images_df: pd.DataFrame, done_source_ids: List[int]
+ sources_df: pd.DataFrame,
+ cfg_err_ra: float,
+ cfg_err_dec: float,
+ p_run: Run,
+ extr_df: pd.DataFrame,
+ min_sigma: float,
+ edge_buffer: float,
+ cluster_threshold: float,
+ allow_nan: bool,
+ add_mode: bool,
+ done_images_df: pd.DataFrame,
+ done_source_ids: List[int],
) -> Tuple[pd.DataFrame, int]:
"""
Check and extract expected measurements, and associated them with the
@@ -533,45 +525,59 @@ def forced_extraction(
The `sources_df` with the extracted sources added.
The total number of forced measurements present in the run.
"""
- logger.info(
- 'Starting force extraction step.'
- )
+ logger.info("Starting force extraction step.")
timer = StopWatch()
# get all the skyregions and related images
cols = [
- 'id', 'name', 'measurements_path', 'path', 'noise_path',
- 'beam_bmaj', 'beam_bmin', 'beam_bpa', 'background_path',
- 'rms_min', 'datetime', 'skyreg__centre_ra',
- 'skyreg__centre_dec', 'skyreg__xtr_radius'
+ "id",
+ "name",
+ "measurements_path",
+ "path",
+ "noise_path",
+ "beam_bmaj",
+ "beam_bmin",
+ "beam_bpa",
+ "background_path",
+ "rms_min",
+ "datetime",
+ "skyreg__centre_ra",
+ "skyreg__centre_dec",
+ "skyreg__xtr_radius",
]
- images_df = pd.DataFrame(list(
- Image.objects.filter(
- run=p_run
- ).select_related('skyreg').order_by('datetime').values(*tuple(cols))
- )).set_index('name')
-# | name | id | measurements_path | path | noise_path |
-# |:------------------------------|-----:|:--------------------|:-------------|:-------------|
-# | VAST_2118-06A.EPOCH01.I.fits | 1 | path/to/file | path/to/file | path/to/file |
-# | VAST_2118-06A.EPOCH03x.I.fits | 3 | path/to/file | path/to/file | path/to/file |
-# | VAST_2118-06A.EPOCH02.I.fits | 2 | path/to/file | path/to/file | path/to/file |
-
-# | name | beam_bmaj | beam_bmin | beam_bpa | background_path |
-# |:------------------------------|------------:|------------:|-----------:|:------------------|
-# | VAST_2118-06A.EPOCH01.I.fits | 0.00589921 | 0.00326088 | -70.4032 | path/to/file |
-# | VAST_2118-06A.EPOCH03x.I.fits | 0.00470991 | 0.00300502 | -83.1128 | path/to/file |
-# | VAST_2118-06A.EPOCH02.I.fits | 0.00351331 | 0.00308565 | 77.2395 | path/to/file |
-
-# | name | rms_min | datetime | skyreg__centre_ra | skyreg__centre_dec | skyreg__xtr_radius |
-# |:------------------------------|----------:|:---------------------------------|--------------------:|---------------------:|---------------------:|
-# | VAST_2118-06A.EPOCH01.I.fits | 0.173946 | 2019-08-27 18:12:16.700000+00:00 | 319.652 | -6.2989 | 6.7401 |
-# | VAST_2118-06A.EPOCH03x.I.fits | 0.165395 | 2019-10-29 10:01:20.500000+00:00 | 319.652 | -6.2989 | 6.7401 |
-# | VAST_2118-06A.EPOCH02.I.fits | 0.16323 | 2019-10-30 08:31:20.200000+00:00 | 319.652 | -6.2989 | 6.7401 |
+ images_df = pd.DataFrame(
+ list(
+ Image.objects.filter(run=p_run)
+ .select_related("skyreg")
+ .order_by("datetime")
+ .values(*tuple(cols))
+ )
+ ).set_index("name")
+
+ # Also set id to str
+ images_df["id"] = images_df["id"].astype(str)
+ # | name | id | measurements_path | path | noise_path |
+ # |:------------------------------|-----:|:--------------------|:-------------|:-------------|
+ # | VAST_2118-06A.EPOCH01.I.fits | 1 | path/to/file | path/to/file | path/to/file |
+ # | VAST_2118-06A.EPOCH03x.I.fits | 3 | path/to/file | path/to/file | path/to/file |
+ # | VAST_2118-06A.EPOCH02.I.fits | 2 | path/to/file | path/to/file | path/to/file |
+
+ # | name | beam_bmaj | beam_bmin | beam_bpa | background_path |
+ # |:------------------------------|------------:|------------:|-----------:|:------------------|
+ # | VAST_2118-06A.EPOCH01.I.fits | 0.00589921 | 0.00326088 | -70.4032 | path/to/file |
+ # | VAST_2118-06A.EPOCH03x.I.fits | 0.00470991 | 0.00300502 | -83.1128 | path/to/file |
+ # | VAST_2118-06A.EPOCH02.I.fits | 0.00351331 | 0.00308565 | 77.2395 | path/to/file |
+
+ # | name | rms_min | datetime | skyreg__centre_ra | skyreg__centre_dec | skyreg__xtr_radius |
+ # |:------------------------------|----------:|:---------------------------------|--------------------:|---------------------:|---------------------:|
+ # | VAST_2118-06A.EPOCH01.I.fits | 0.173946 | 2019-08-27 18:12:16.700000+00:00 | 319.652 | -6.2989 | 6.7401 |
+ # | VAST_2118-06A.EPOCH03x.I.fits | 0.165395 | 2019-10-29 10:01:20.500000+00:00 | 319.652 | -6.2989 | 6.7401 |
+ # | VAST_2118-06A.EPOCH02.I.fits | 0.16323 | 2019-10-30 08:31:20.200000+00:00 | 319.652 | -6.2989 | 6.7401 |
# Explode out the img_diff column.
- extr_df = extr_df.explode('img_diff').reset_index()
+ extr_df = extr_df.explode("img_diff").reset_index()
total_to_extract = extr_df.shape[0]
if add_mode:
@@ -586,11 +592,11 @@ def forced_extraction(
extr_df = pd.concat(
[
- extr_df[~extr_df['img_diff'].isin(done_images_df['name'])],
+ extr_df[~extr_df["img_diff"].isin(done_images_df["name"])],
extr_df[
- (~extr_df['source'].isin(done_source_ids))
- & (extr_df['img_diff'].isin(done_images_df.name))
- ]
+ (~extr_df["source"].isin(done_source_ids))
+ & (extr_df["img_diff"].isin(done_images_df.name))
+ ],
]
).sort_index()
@@ -601,119 +607,97 @@ def forced_extraction(
timer.reset()
extr_df = parallel_extraction(
- extr_df, images_df, sources_df[['source', 'image', 'flux_peak']],
- min_sigma, edge_buffer, cluster_threshold, allow_nan, add_mode,
- p_run.path
- )
- logger.info(
- 'Force extraction step time: %.2f seconds', timer.reset()
+ extr_df,
+ images_df,
+ sources_df[["source", "image", "flux_peak"]],
+ min_sigma,
+ edge_buffer,
+ cluster_threshold,
+ allow_nan,
+ add_mode,
+ p_run.path,
)
+ logger.info("Force extraction step time: %.2f seconds", timer.reset())
# make measurement names unique for db constraint
- extr_df['name'] = extr_df['name'] + f'_f_run{p_run.id:06d}'
+ extr_df["name"] = extr_df["name"] + f"_f_{str(p_run.id).split('-')[0]}"
# select sensible flux values and set the columns with fix values
- values = {
- 'flux_int': 0,
- 'flux_int_err': 0
- }
+ values = {"flux_int": 0, "flux_int_err": 0}
extr_df = extr_df.fillna(value=values)
extr_df = extr_df[
- (extr_df['flux_int'] != 0)
- & (extr_df['flux_int_err'] != 0)
- & (extr_df['chi_squared_fit'] != np.inf)
- & (extr_df['chi_squared_fit'] != np.nan)
+ (extr_df["flux_int"] != 0)
+ & (extr_df["flux_int_err"] != 0)
+ & (extr_df["chi_squared_fit"] != np.inf)
+ & (extr_df["chi_squared_fit"] != np.nan)
]
- default_pos_err = settings.POS_DEFAULT_MIN_ERROR / 3600.
- extr_df['ra_err'] = default_pos_err
- extr_df['dec_err'] = default_pos_err
- extr_df['err_bmaj'] = 0.
- extr_df['err_bmin'] = 0.
- extr_df['err_pa'] = 0.
- extr_df['ew_sys_err'] = cfg_err_ra
- extr_df['ns_sys_err'] = cfg_err_dec
- extr_df['error_radius'] = 0.
-
- extr_df['uncertainty_ew'] = np.hypot(
- cfg_err_ra,
- default_pos_err
- )
- extr_df['weight_ew'] = 1. / extr_df['uncertainty_ew'].values**2
- extr_df['uncertainty_ns'] = np.hypot(
- cfg_err_dec,
- default_pos_err
- )
- extr_df['weight_ns'] = 1. / extr_df['uncertainty_ns'].values**2
-
- extr_df['flux_peak'] = extr_df['flux_int']
- extr_df['flux_peak_err'] = extr_df['flux_int_err']
- extr_df['local_rms'] = extr_df['flux_int_err']
- extr_df['snr'] = (
- extr_df['flux_peak'].values
- / extr_df['local_rms'].values
- )
- extr_df['spectral_index'] = 0.
- extr_df['dr'] = 0.
- extr_df['d2d'] = 0.
- extr_df['forced'] = True
- extr_df['compactness'] = 1.
- extr_df['psf_bmaj'] = extr_df['bmaj']
- extr_df['psf_bmin'] = extr_df['bmin']
- extr_df['psf_pa'] = extr_df['pa']
- extr_df['flag_c4'] = False
- extr_df['spectral_index_from_TT'] = False
- extr_df['has_siblings'] = False
- extr_df['flux_int_isl_ratio'] = 1.0
- extr_df['flux_peak_isl_ratio'] = 1.0
-
- col_order = read_schema(
- images_df.iloc[0]['measurements_path']
- ).names
- col_order.remove('id')
+ default_pos_err = settings.POS_DEFAULT_MIN_ERROR / 3600.0
+ extr_df["ra_err"] = default_pos_err
+ extr_df["dec_err"] = default_pos_err
+ extr_df["err_bmaj"] = 0.0
+ extr_df["err_bmin"] = 0.0
+ extr_df["err_pa"] = 0.0
+ extr_df["ew_sys_err"] = cfg_err_ra
+ extr_df["ns_sys_err"] = cfg_err_dec
+ extr_df["error_radius"] = 0.0
+
+ extr_df["uncertainty_ew"] = np.hypot(cfg_err_ra, default_pos_err)
+ extr_df["weight_ew"] = 1.0 / extr_df["uncertainty_ew"].values ** 2
+ extr_df["uncertainty_ns"] = np.hypot(cfg_err_dec, default_pos_err)
+ extr_df["weight_ns"] = 1.0 / extr_df["uncertainty_ns"].values ** 2
+
+ extr_df["flux_peak"] = extr_df["flux_int"]
+ extr_df["flux_peak_err"] = extr_df["flux_int_err"]
+ extr_df["local_rms"] = extr_df["flux_int_err"]
+ extr_df["snr"] = extr_df["flux_peak"].values / extr_df["local_rms"].values
+ extr_df["spectral_index"] = 0.0
+ extr_df["dr"] = 0.0
+ extr_df["d2d"] = 0.0
+ extr_df["forced"] = True
+ extr_df["compactness"] = 1.0
+ extr_df["psf_bmaj"] = extr_df["bmaj"]
+ extr_df["psf_bmin"] = extr_df["bmin"]
+ extr_df["psf_pa"] = extr_df["pa"]
+ extr_df["flag_c4"] = False
+ extr_df["spectral_index_from_TT"] = False
+ extr_df["has_siblings"] = False
+ extr_df["flux_int_isl_ratio"] = 1.0
+ extr_df["flux_peak_isl_ratio"] = 1.0
+
+ col_order = read_schema(images_df.iloc[0]["measurements_path"]).names
remaining = list(set(extr_df.columns) - set(col_order))
extr_df = extr_df[col_order + remaining]
- # upload the measurements, a column 'id' is returned with the DB id
- extr_df = make_upload_measurements(extr_df)
+ # upload the measurements
+ make_upload_measurements(extr_df)
- extr_df = extr_df.rename(columns={'source_tmp_id': 'source'})
+ extr_df = extr_df.rename(columns={"source_tmp_id": "source"})
# write forced measurements to specific parquet
- logger.info(
- 'Saving forced measurements to specific parquet file...'
- )
+ logger.info("Saving forced measurements to specific parquet file...")
parallel_write_parquet(extr_df, p_run.path, add_mode)
# Required to rename this column for the image add mode.
- extr_df = extr_df.rename(columns={'time': 'datetime'})
+ extr_df = extr_df.rename(columns={"time": "datetime"})
# append new meas into main df and proceed with source groupby etc
sources_df = pd.concat(
- [
- sources_df,
- extr_df.loc[:, extr_df.columns.isin(sources_df.columns)]
- ],
- ignore_index=True
+ [sources_df, extr_df.loc[:, extr_df.columns.isin(sources_df.columns)]],
+ ignore_index=True,
)
# get the number of forced extractions for the run
- forced_parquets = glob(
- os.path.join(p_run.path, "forced_measurements*.parquet"))
+ forced_parquets = glob(os.path.join(p_run.path, "forced_measurements*.parquet"))
if forced_parquets:
n_forced = (
- dd.read_parquet(forced_parquets, columns=['id'])
- .count()
- .compute()
- .values[0]
+ dd.read_parquet(forced_parquets, columns=["id"]).count().compute().values[0]
)
else:
n_forced = 0
- logger.info(
- 'Total forced extraction time: %.2f seconds', timer.reset_init()
- )
+ logger.info("Total forced extraction time: %.2f seconds", timer.reset_init())
return sources_df, n_forced
diff --git a/vast_pipeline/pipeline/loading.py b/vast_pipeline/pipeline/loading.py
index a7d8f459e..c36568d15 100644
--- a/vast_pipeline/pipeline/loading.py
+++ b/vast_pipeline/pipeline/loading.py
@@ -15,8 +15,14 @@
association_models_generator,
)
from vast_pipeline.models import (
- Association, Band, Measurement, SkyRegion, Source, RelatedSource,
- Run, Image
+ Association,
+ Band,
+ Measurement,
+ SkyRegion,
+ Source,
+ RelatedSource,
+ Run,
+ Image,
)
from vast_pipeline.pipeline.utils import get_create_img, get_create_img_band
from vast_pipeline.utils.utils import StopWatch
@@ -29,9 +35,9 @@
def bulk_upload_model(
djmodel: models.Model,
generator: Iterable[Generator[models.Model, None, None]],
- batch_size: int=10_000, return_ids: bool=False
+ batch_size: int = 10_000,
) -> List[int]:
- '''
+ """
Bulk upload a list of generator objects of django models to db.
Args:
@@ -41,33 +47,23 @@ def bulk_upload_model(
The generator objects of the model to upload.
batch_size:
How many records to upload at once.
- return_ids:
- When set to True, the database IDs of the uploaded objects are
- returned.
Returns:
None or a list of the database IDs of the uploaded objects.
- '''
- bulk_ids = []
+ """
while True:
items = list(islice(generator, batch_size))
if not items:
break
out_bulk = djmodel.objects.bulk_create(items)
- logger.info('Bulk created #%i %s', len(out_bulk), djmodel.__name__)
- # save the DB ids to return
- if return_ids:
- bulk_ids.extend(list(map(lambda i: i.id, out_bulk)))
-
- if return_ids:
- return bulk_ids
+ logger.info("Bulk created #%i %s", len(out_bulk), djmodel.__name__)
def make_upload_images(
paths: Dict[str, Dict[str, str]], image_config: Dict
) -> Tuple[List[Image], List[SkyRegion], List[Band]]:
- '''
+ """
Carry the first part of the pipeline, by uploading all the images
to the image table and populated band and skyregion objects.
@@ -84,20 +80,16 @@ def make_upload_images(
A list of Image objects that have been uploaded.
A list of SkyRegion objects that have been uploaded.
A list of Band objects that have been uploaded.
- '''
+ """
timer = StopWatch()
images = []
skyregions = []
bands = []
- for path in paths['selavy']:
+ for path in paths["selavy"]:
# STEP #1: Load image and measurements
- image = SelavyImage(
- path,
- paths,
- image_config
- )
- logger.info('Reading image %s ...', image.name)
+ image = SelavyImage(path, paths, image_config)
+ logger.info("Reading image %s ...", image.name)
# 1.1 get/create the frequency band
with transaction.atomic():
@@ -128,7 +120,7 @@ def make_upload_images(
)
# upload measurements, a column with the db is added to the df
- measurements = make_upload_measurements(measurements)
+ make_upload_measurements(measurements)
# save measurements to parquet file in pipeline run folder
base_folder = os.path.dirname(img.measurements_path)
@@ -138,10 +130,7 @@ def make_upload_images(
measurements.to_parquet(img.measurements_path, index=False)
del measurements, image, band, img
- logger.info(
- 'Total images upload/loading time: %.2f seconds',
- timer.reset_init()
- )
+ logger.info("Total images upload/loading time: %.2f seconds", timer.reset_init())
return images, skyregions, bands
@@ -149,7 +138,7 @@ def make_upload_images(
def make_upload_sources(
sources_df: pd.DataFrame, pipeline_run: Run, add_mode: bool = False
) -> pd.DataFrame:
- '''
+ """
Delete previous sources for given pipeline run and bulk upload
new found sources as well as related sources.
@@ -164,30 +153,26 @@ def make_upload_sources(
Returns:
The input dataframe with the 'id' column added.
- '''
+ """
# create sources in DB
with transaction.atomic():
- if (add_mode is False and
- Source.objects.filter(run=pipeline_run).exists()):
- logger.info('Removing objects from previous pipeline run')
- n_del, detail_del = (
- Source.objects.filter(run=pipeline_run).delete()
- )
+ if add_mode is False and Source.objects.filter(run=pipeline_run).exists():
+ logger.info("Removing objects from previous pipeline run")
+ n_del, detail_del = Source.objects.filter(run=pipeline_run).delete()
logger.info(
- ('Deleting all sources and related objects for this run. '
- 'Total objects deleted: %i'),
+ (
+ "Deleting all sources and related objects for this run. "
+ "Total objects deleted: %i"
+ ),
n_del,
)
- logger.debug('(type, #deleted): %s', detail_del)
+ logger.debug("(type, #deleted): %s", detail_del)
- src_dj_ids = bulk_upload_model(
+ bulk_upload_model(
Source,
source_models_generator(sources_df, pipeline_run=pipeline_run),
- return_ids=True
)
- sources_df['id'] = src_dj_ids
-
return sources_df
@@ -219,13 +204,11 @@ def make_upload_associations(associations_df: pd.DataFrame) -> None:
Returns:
None.
"""
- logger.info('Upload associations...')
- bulk_upload_model(
- Association, association_models_generator(associations_df)
- )
+ logger.info("Upload associations...")
+ bulk_upload_model(Association, association_models_generator(associations_df))
-def make_upload_measurements(measurements_df: pd.DataFrame) -> pd.DataFrame:
+def make_upload_measurements(measurements_df: pd.DataFrame) -> None:
"""
Uploads the measurements from the supplied measurements DataFrame.
@@ -237,21 +220,11 @@ def make_upload_measurements(measurements_df: pd.DataFrame) -> pd.DataFrame:
Returns:
Original DataFrame with the database ID attached to each row.
"""
- meas_dj_ids = bulk_upload_model(
- Measurement,
- measurement_models_generator(measurements_df),
- return_ids=True
- )
-
- measurements_df['id'] = meas_dj_ids
+ bulk_upload_model(Measurement, measurement_models_generator(measurements_df))
- return measurements_df
-
-def update_sources(
- sources_df: pd.DataFrame, batch_size: int = 10_000
-) -> pd.DataFrame:
- '''
+def update_sources(sources_df: pd.DataFrame, batch_size: int = 10_000) -> pd.DataFrame:
+ """
Update database using SQL code. This function opens one connection to the
database, and closes it after the update is done.
@@ -267,37 +240,36 @@ def update_sources(
Returns:
DataFrame containing the new data to be uploaded to the database.
- '''
+ """
# Get all possible columns from the model
all_source_table_cols = [
- fld.attname for fld in Source._meta.get_fields()
- if getattr(fld, 'attname', None) is not None
+ fld.attname
+ for fld in Source._meta.get_fields()
+ if getattr(fld, "attname", None) is not None
]
# Filter to those present in sources_df
- columns = [
- col for col in all_source_table_cols if col in sources_df.columns
- ]
+ columns = [col for col in all_source_table_cols if col in sources_df.columns]
- sources_df['id'] = sources_df.index.values
+ sources_df["id"] = sources_df.index.values
- batches = np.ceil(len(sources_df)/batch_size)
+ batches = np.ceil(len(sources_df) / batch_size)
dfs = np.array_split(sources_df, batches)
with connection.cursor() as cursor:
for df_batch in dfs:
- SQL_comm = SQL_update(
- df_batch, Source, index='id', columns=columns
- )
+ SQL_comm = SQL_update(df_batch, Source, index="id", columns=columns)
cursor.execute(SQL_comm)
return sources_df
def SQL_update(
- df: pd.DataFrame, model: models.Model, index: Optional[str] = None,
- columns: Optional[List[str]] = None
+ df: pd.DataFrame,
+ model: models.Model,
+ index: Optional[str] = None,
+ columns: Optional[List[str]] = None,
) -> str:
- '''
+ """
Generate the SQL code required to update the database.
Args:
@@ -316,7 +288,7 @@ def SQL_update(
Returns:
The SQL command to update the database.
- '''
+ """
# set index and columns if None
if index is None:
index = model._meta.pk.name
@@ -326,8 +298,8 @@ def SQL_update(
# get names
table = model._meta.db_table
- new_columns = ', '.join('new_'+c for c in columns)
- set_columns = ', '.join(c+'=new_'+c for c in columns)
+ new_columns = ", ".join("new_" + c for c in columns)
+ set_columns = ", ".join(c + "=new_" + c for c in columns)
# get index values and new values
column_headers = [index]
@@ -335,9 +307,9 @@ def SQL_update(
data_arr = df[column_headers].to_numpy()
values = []
for row in data_arr:
- val_row = '(' + ', '.join(f'{val}' for val in row) + ')'
+ val_row = "(" + f"'{row[0]}', " + ", ".join(f"{val}" for val in row[1:]) + ")"
values.append(val_row)
- values = ', '.join(values)
+ values = ", ".join(values)
# update database
SQL_comm = f"""
@@ -345,7 +317,7 @@ def SQL_update(
SET {set_columns}
FROM (VALUES {values})
AS new_values (index_col, {new_columns})
- WHERE {index}=index_col;
+ WHERE {index}=index_col::uuid;
"""
return SQL_comm
diff --git a/vast_pipeline/pipeline/main.py b/vast_pipeline/pipeline/main.py
index 6e1fe1721..5fa55e1d5 100644
--- a/vast_pipeline/pipeline/main.py
+++ b/vast_pipeline/pipeline/main.py
@@ -28,7 +28,7 @@
get_src_skyregion_merged_df,
group_skyregions,
get_parallel_assoc_image_df,
- write_parquets
+ write_parquets,
)
from .errors import MaxPipelineRunsError
@@ -37,7 +37,7 @@
logger = logging.getLogger(__name__)
-class Pipeline():
+class Pipeline:
"""Instance of a pipeline. All the methods runs the pipeline opearations, such as
association.
@@ -53,6 +53,7 @@ class Pipeline():
previous_parquets: A dict mapping that provides the paths to parquet files for
previous executions of this pipeline run.
"""
+
def __init__(self, name: str, config_path: str, validate_config: bool = True):
"""Initialise an instance of Pipeline with a name and configuration file path.
@@ -67,9 +68,9 @@ def __init__(self, name: str, config_path: str, validate_config: bool = True):
config_path, validate=validate_config
)
self.img_paths: Dict[str, Dict[str, str]] = {
- 'selavy': {},
- 'noise': {},
- 'background': {},
+ "selavy": {},
+ "noise": {},
+ "background": {},
} # maps input image paths to their selavy/noise/background counterpart path
self.img_epochs: Dict[str, str] = {} # maps image names to their provided epoch
self.add_mode: bool = False
@@ -112,9 +113,9 @@ def process_pipeline(self, p_run: Run) -> None:
Returns:
None
"""
- logger.info(f'Epoch based association: {self.config.epoch_based}')
+ logger.info(f"Epoch based association: {self.config.epoch_based}")
if self.add_mode:
- logger.info('Running in image add mode.')
+ logger.info("Running in image add mode.")
# Update epoch based flag to not cause user confusion when running
# the pipeline (i.e. if it was only updated at the end). It is not
@@ -130,8 +131,7 @@ def process_pipeline(self, p_run: Run) -> None:
# upload/retrieve image data
images, skyregions, bands = make_upload_images(
- self.img_paths,
- self.config.image_opts()
+ self.img_paths, self.config.image_opts()
)
# associate the pipeline run with each image
@@ -140,11 +140,13 @@ def process_pipeline(self, p_run: Run) -> None:
add_run_to_img(p_run, img)
# write parquet files and retrieve skyregions as a dataframe
- skyregs_df = write_parquets(images, skyregions, bands, self.config["run"]["path"])
+ skyregs_df = write_parquets(
+ images, skyregions, bands, self.config["run"]["path"]
+ )
# STEP #2: measurements association
# order images by time
- images.sort(key=operator.attrgetter('datetime'))
+ images.sort(key=operator.attrgetter("datetime"))
# If the user has given lists we need to reorder the
# image epochs such that they are in date order.
@@ -153,9 +155,7 @@ def process_pipeline(self, p_run: Run) -> None:
for i, img in enumerate(images):
self.img_epochs[img.name] = i + 1
- image_epochs = [
- self.img_epochs[img.name] for img in images
- ]
+ image_epochs = [self.img_epochs[img.name] for img in images]
limit = Angle(self.config["source_association"]["radius"] * u.arcsec)
dr_limit = self.config["source_association"]["deruiter_radius"]
bw_limit = self.config["source_association"]["deruiter_beamwidth_limit"]
@@ -166,20 +166,17 @@ def process_pipeline(self, p_run: Run) -> None:
# 2.1 Check if sky regions to be associated can be
# split into connected point groups
skyregion_groups = group_skyregions(
- skyregs_df[['id', 'centre_ra', 'centre_dec', 'xtr_radius']]
+ skyregs_df[["id", "centre_ra", "centre_dec", "xtr_radius"]]
)
- n_skyregion_groups = skyregion_groups[
- 'skyreg_group'
- ].unique().shape[0]
+ n_skyregion_groups = skyregion_groups["skyreg_group"].unique().shape[0]
# Get already done images if in add mode
if self.add_mode:
done_images_df = pd.read_parquet(
- self.previous_parquets['images'], columns=['id', 'name']
+ self.previous_parquets["images"], columns=["id", "name"]
)
done_source_ids = pd.read_parquet(
- self.previous_parquets['sources'],
- columns=['wavg_ra']
+ self.previous_parquets["sources"], columns=["wavg_ra"]
).index.tolist()
else:
done_images_df = None
@@ -187,10 +184,8 @@ def process_pipeline(self, p_run: Run) -> None:
# 2.2 Associate with other measurements
if self.config["source_association"]["parallel"] and n_skyregion_groups > 1:
- images_df = get_parallel_assoc_image_df(
- images, skyregion_groups
- )
- images_df['epoch'] = image_epochs
+ images_df = get_parallel_assoc_image_df(images, skyregion_groups)
+ images_df["epoch"] = image_epochs
sources_df = parallel_association(
images_df,
@@ -203,23 +198,18 @@ def process_pipeline(self, p_run: Run) -> None:
self.add_mode,
self.previous_parquets,
done_images_df,
- done_source_ids
+ done_source_ids,
)
else:
images_df = pd.DataFrame.from_dict(
- {
- 'image_dj': images,
- 'epoch': image_epochs
- }
+ {"image_dj": images, "epoch": image_epochs}
)
- images_df['skyreg_id'] = images_df['image_dj'].apply(
- lambda x: x.skyreg_id
+ images_df["skyreg_id"] = images_df["image_dj"].apply(
+ lambda x: str(x.skyreg_id)
)
- images_df['image_name'] = images_df['image_dj'].apply(
- lambda x: x.name
- )
+ images_df["image_name"] = images_df["image_dj"].apply(lambda x: x.name)
sources_df = association(
images_df,
@@ -230,24 +220,30 @@ def process_pipeline(self, p_run: Run) -> None:
self.config,
self.add_mode,
self.previous_parquets,
- done_images_df
+ done_images_df,
)
# Obtain the number of selavy measurements for the run
# n_selavy_measurements = sources_df.
- nr_selavy_measurements = sources_df['id'].unique().shape[0]
+ nr_selavy_measurements = sources_df["id"].unique().shape[0]
# STEP #3: Merge sky regions and sources ready for
# steps 4 and 5 below.
missing_source_cols = [
- 'source', 'datetime', 'image', 'epoch',
- 'interim_ew', 'weight_ew', 'interim_ns', 'weight_ns'
+ "source",
+ "datetime",
+ "image",
+ "epoch",
+ "interim_ew",
+ "weight_ew",
+ "interim_ns",
+ "weight_ns",
]
# need to make sure no forced measurments are being passed which
# could happen in add mode, otherwise the wrong detection image is
# assigned.
missing_sources_df = get_src_skyregion_merged_df(
- sources_df.loc[sources_df['forced'] == False, missing_source_cols],
+ sources_df.loc[sources_df["forced"] == False, missing_source_cols],
images_df,
skyregs_df,
)
@@ -258,23 +254,18 @@ def process_pipeline(self, p_run: Run) -> None:
missing_sources_df,
self.config["new_sources"]["min_sigma"],
self.config["source_monitoring"]["edge_buffer_scale"],
- p_run
+ p_run,
)
# Drop column no longer required in missing_sources_df.
- missing_sources_df = (
- missing_sources_df.drop(['in_primary'], axis=1)
- )
+ missing_sources_df = missing_sources_df.drop(["in_primary"], axis=1)
# STEP #5: Run forced extraction/photometry if asked
if self.config["source_monitoring"]["monitor"]:
- (
+ (sources_df, nr_forced_measurements) = forced_extraction(
sources_df,
- nr_forced_measurements
- ) = forced_extraction(
- sources_df,
- self.config["measurements"]["ra_uncertainty"] / 3600.,
- self.config["measurements"]["dec_uncertainty"] / 3600.,
+ self.config["measurements"]["ra_uncertainty"] / 3600.0,
+ self.config["measurements"]["dec_uncertainty"] / 3600.0,
p_run,
missing_sources_df,
self.config["source_monitoring"]["min_sigma"],
@@ -283,7 +274,7 @@ def process_pipeline(self, p_run: Run) -> None:
self.config["source_monitoring"]["allow_nan"],
self.add_mode,
done_images_df,
- done_source_ids
+ done_source_ids,
)
del missing_sources_df
@@ -298,7 +289,7 @@ def process_pipeline(self, p_run: Run) -> None:
self.config["variability"]["source_aggregate_pair_metrics_min_abs_vs"],
self.add_mode,
done_source_ids,
- self.previous_parquets
+ self.previous_parquets,
)
# calculate number processed images
@@ -310,7 +301,9 @@ def process_pipeline(self, p_run: Run) -> None:
p_run.n_sources = nr_sources
p_run.n_selavy_measurements = nr_selavy_measurements
p_run.n_forced_measurements = (
- nr_forced_measurements if self.config["source_monitoring"]["monitor"] else 0
+ nr_forced_measurements
+ if self.config["source_monitoring"]["monitor"]
+ else 0
)
p_run.n_new_sources = nr_new_sources
p_run.save()
@@ -334,7 +327,7 @@ def check_current_runs() -> None:
raise MaxPipelineRunsError
@staticmethod
- def set_status(pipe_run: Run, status: str=None) -> None:
+ def set_status(pipe_run: Run, status: str = None) -> None:
"""
Function to change the status of a pipeline run model object and save
to the database.
@@ -346,8 +339,8 @@ def set_status(pipe_run: Run, status: str=None) -> None:
Returns:
None
"""
- #TODO: This function gives no feedback if the status is not accepted?
- choices = [x[0] for x in Run._meta.get_field('status').choices]
+ # TODO: This function gives no feedback if the status is not accepted?
+ choices = [x[0] for x in Run._meta.get_field("status").choices]
if status and status in choices and pipe_run.status != status:
with transaction.atomic():
pipe_run.status = status
diff --git a/vast_pipeline/pipeline/model_generator.py b/vast_pipeline/pipeline/model_generator.py
index 310babae0..d72c59388 100644
--- a/vast_pipeline/pipeline/model_generator.py
+++ b/vast_pipeline/pipeline/model_generator.py
@@ -6,16 +6,14 @@
from typing import Iterable, Generator
from vast_pipeline.utils.utils import deg2hms, deg2dms
-from vast_pipeline.models import (
- Association, Measurement, Source, RelatedSource, Run
-)
+from vast_pipeline.models import Association, Measurement, Source, RelatedSource, Run
logger = logging.getLogger(__name__)
def measurement_models_generator(
- meas_df: pd.DataFrame
+ meas_df: pd.DataFrame,
) -> Iterable[Generator[Measurement, None, None]]:
"""
Creates a generator object containing yielded Measurement objects from
@@ -33,7 +31,7 @@ def measurement_models_generator(
for i, row in meas_df.iterrows():
one_m = Measurement()
for fld in one_m._meta.get_fields():
- if getattr(fld, 'attname', None) and fld.attname in row.index:
+ if getattr(fld, "attname", None) and fld.attname in row.index:
setattr(one_m, fld.attname, row[fld.attname])
yield one_m
@@ -65,15 +63,16 @@ def source_models_generator(
src = Source()
src.run_id = pipeline_run.id
src.name = name
+ src.id = i
for fld in src._meta.get_fields():
- if getattr(fld, 'attname', None) and fld.attname in row.index:
+ if getattr(fld, "attname", None) and fld.attname in row.index:
setattr(src, fld.attname, row[fld.attname])
yield src
def association_models_generator(
- assoc_df: pd.DataFrame
+ assoc_df: pd.DataFrame,
) -> Iterable[Generator[Association, None, None]]:
"""
Creates a generator object containing yielded Association objects from
@@ -89,15 +88,15 @@ def association_models_generator(
"""
for i, row in assoc_df.iterrows():
yield Association(
- meas_id=row['id'],
- source_id=row['source_id'],
- d2d=row['d2d'],
- dr=row['dr'],
+ meas_id=row["id"],
+ source_id=row["source"],
+ d2d=row["d2d"],
+ dr=row["dr"],
)
def related_models_generator(
- related_df: pd.DataFrame
+ related_df: pd.DataFrame,
) -> Iterable[Generator[RelatedSource, None, None]]:
"""
Creates a generator object containing yielded Association objects from
diff --git a/vast_pipeline/pipeline/pairs.py b/vast_pipeline/pipeline/pairs.py
index 470cbab3b..0e2863ccb 100644
--- a/vast_pipeline/pipeline/pairs.py
+++ b/vast_pipeline/pipeline/pairs.py
@@ -89,7 +89,9 @@ def calculate_measurement_pair_metrics(df: pd.DataFrame) -> pd.DataFrame:
dd.from_pandas(df, n_cpu)
.groupby("source")["id"]
.apply(
- lambda x: pd.DataFrame(list(combinations(x, 2))), meta={0: "i", 1: "i"},)
+ lambda x: pd.DataFrame(list(combinations(x, 2))),
+ meta={0: "i", 1: "i"},
+ )
.compute(num_workers=n_cpu, scheduler="processes")
)
@@ -109,40 +111,46 @@ def calculate_measurement_pair_metrics(df: pd.DataFrame) -> pd.DataFrame:
33644 11128 6216 23534
Where source is the source ID, id_a and id_b are measurement IDs.
"""
- measurement_combinations = measurement_combinations.reset_index(
- level=1, drop=True
- ).rename(columns={0: "id_a", 1: "id_b"}).astype(int).reset_index()
+ measurement_combinations = (
+ measurement_combinations.reset_index(level=1, drop=True)
+ .rename(columns={0: "id_a", 1: "id_b"})
+ .astype(str)
+ .reset_index()
+ )
# Dask has a tendency to swap which order the measurement pairs are
# defined in, even if the dataframe is pre-sorted. We want the pairs to be
# in date order (a < b) so the code below corrects any that are not.
measurement_combinations = measurement_combinations.join(
- df[['source', 'id', 'datetime']].set_index(['source', 'id']),
- on=['source', 'id_a'],
+ df[["source", "id", "datetime"]].set_index(["source", "id"]),
+ on=["source", "id_a"],
)
measurement_combinations = measurement_combinations.join(
- df[['source', 'id', 'datetime']].set_index(['source', 'id']),
- on=['source', 'id_b'], lsuffix='_a', rsuffix='_b'
+ df[["source", "id", "datetime"]].set_index(["source", "id"]),
+ on=["source", "id_b"],
+ lsuffix="_a",
+ rsuffix="_b",
)
to_correct_mask = (
- measurement_combinations['datetime_a']
- > measurement_combinations['datetime_b']
+ measurement_combinations["datetime_a"] > measurement_combinations["datetime_b"]
)
if np.any(to_correct_mask):
- logger.debug('Correcting measurement pairs order')
+ logger.debug("Correcting measurement pairs order")
(
- measurement_combinations.loc[to_correct_mask, 'id_a'],
- measurement_combinations.loc[to_correct_mask, 'id_b']
- ) = np.array([
- measurement_combinations.loc[to_correct_mask, 'id_b'].values,
- measurement_combinations.loc[to_correct_mask, 'id_a'].values
- ])
+ measurement_combinations.loc[to_correct_mask, "id_a"],
+ measurement_combinations.loc[to_correct_mask, "id_b"],
+ ) = np.array(
+ [
+ measurement_combinations.loc[to_correct_mask, "id_b"].values,
+ measurement_combinations.loc[to_correct_mask, "id_a"].values,
+ ]
+ )
measurement_combinations = measurement_combinations.drop(
- ['datetime_a', 'datetime_b'], axis=1
+ ["datetime_a", "datetime_b"], axis=1
)
# add the measurement fluxes and errors
diff --git a/vast_pipeline/pipeline/utils.py b/vast_pipeline/pipeline/utils.py
index 82437141c..924e5a345 100644
--- a/vast_pipeline/pipeline/utils.py
+++ b/vast_pipeline/pipeline/utils.py
@@ -7,6 +7,7 @@
import logging
import glob
import shutil
+import uuid
import numpy as np
import pandas as pd
import pyarrow as pa
@@ -24,11 +25,12 @@
from vast_pipeline.image.main import FitsImage, SelavyImage
from vast_pipeline.image.utils import open_fits
from vast_pipeline.utils.utils import (
- eq_to_cart, StopWatch, optimize_ints, optimize_floats
-)
-from vast_pipeline.models import (
- Band, Image, Run, SkyRegion
+ eq_to_cart,
+ StopWatch,
+ optimize_ints,
+ optimize_floats,
)
+from vast_pipeline.models import Band, Image, Run, SkyRegion
logger = logging.getLogger(__name__)
@@ -36,7 +38,7 @@
def get_create_skyreg(image: Image) -> SkyRegion:
- '''
+ """
This creates a Sky Region object in Django ORM given the related
image object.
@@ -45,18 +47,16 @@ def get_create_skyreg(image: Image) -> SkyRegion:
Returns:
The sky region Django ORM object.
- '''
+ """
# In the calculations below, it is assumed the image has square
# pixels (this pipeline has been designed for ASKAP images, so it
# should always be square). It will likely give wrong results if not
skyregions = SkyRegion.objects.filter(
- centre_ra=image.ra,
- centre_dec=image.dec,
- xtr_radius=image.fov_bmin
+ centre_ra=image.ra, centre_dec=image.dec, xtr_radius=image.fov_bmin
)
if skyregions:
skyr = skyregions.get()
- logger.info('Found sky region %s', skyr)
+ logger.info("Found sky region %s", skyr)
else:
x, y, z = eq_to_cart(image.ra, image.dec)
skyr = SkyRegion(
@@ -70,13 +70,13 @@ def get_create_skyreg(image: Image) -> SkyRegion:
z=z,
)
skyr.save()
- logger.info('Created sky region %s', skyr)
+ logger.info("Created sky region %s", skyr)
return skyr
def get_create_img_band(image: FitsImage) -> Band:
- '''
+ """
Return the existing Band row for the given FitsImage.
An image is considered to belong to a band if its frequency is within some
tolerance of the band's frequency.
@@ -87,12 +87,12 @@ def get_create_img_band(image: FitsImage) -> Band:
Returns:
The band Django ORM object.
- '''
+ """
# For now we match bands using the central frequency.
# This assumes that every band has a unique frequency,
# which is true for the data we've used so far.
- freq = int(image.freq_eff * 1.e-6)
- freq_band = int(image.freq_bw * 1.e-6)
+ freq = int(image.freq_eff * 1.0e-6)
+ freq_band = int(image.freq_bw * 1.0e-6)
# TODO: refine the band query
for band in Band.objects.all():
diff = abs(freq - band.frequency) / float(band.frequency)
@@ -101,7 +101,7 @@ def get_create_img_band(image: FitsImage) -> Band:
# no band has been found so create it
band = Band(name=str(freq), frequency=freq, bandwidth=freq_band)
- logger.info('Adding new frequency band: %s', band)
+ logger.info("Adding new frequency band: %s", band)
band.save()
return band
@@ -131,27 +131,28 @@ def get_create_img(band_id: int, image: SelavyImage) -> Tuple[Image, bool]:
else:
# at this stage, measurement parquet file is not created but
# assume location
- img_folder_name = image.name.replace('.', '_')
+ img_folder_name = image.name.replace(".", "_")
measurements_path = os.path.join(
settings.PIPELINE_WORKING_DIR,
- 'images',
+ "images",
img_folder_name,
- 'measurements.parquet'
- )
- img = Image(
- band_id=band_id,
- measurements_path=measurements_path
+ "measurements.parquet",
)
+ img = Image(band_id=band_id, measurements_path=measurements_path)
# set the attributes and save the image,
# by selecting only valid (not hidden) attributes
# FYI attributs and/or method starting with _ are hidden
# and with __ can't be modified/called
for fld in img._meta.get_fields():
- if getattr(fld, 'attname', None) and (getattr(image, fld.attname, None) is not None):
+ if getattr(fld, "attname", None) and (
+ getattr(image, fld.attname, None) is not None
+ ):
setattr(img, fld.attname, getattr(image, fld.attname))
- img.rms_median, img.rms_min, img.rms_max = get_rms_noise_image_values(img.noise_path)
+ img.rms_median, img.rms_min, img.rms_max = get_rms_noise_image_values(
+ img.noise_path
+ )
# get create the sky region and associate with image
img.skyreg = get_create_skyreg(img)
@@ -163,7 +164,7 @@ def get_create_img(band_id: int, image: SelavyImage) -> Tuple[Image, bool]:
def get_create_p_run(
name: str, path: str, description: str = None, user: User = None
) -> Tuple[Run, bool]:
- '''
+ """
Get or create a pipeline run in db, return the run django object and
a flag True/False if has been created or already exists.
@@ -177,7 +178,7 @@ def get_create_p_run(
Returns:
The pipeline run object.
Whether the pipeline run already existed ('True') or not ('False').
- '''
+ """
p_run = Run.objects.filter(name__exact=name)
if p_run:
return p_run.get(), True
@@ -207,18 +208,16 @@ def add_run_to_img(pipeline_run: Run, img: Image) -> None:
skyreg = img.skyreg
# check and add the many to many if not existent
if not Image.objects.filter(id=img.id, run__id=pipeline_run.id).exists():
- logger.info('Adding %s to image %s', pipeline_run, img.name)
+ logger.info("Adding %s to image %s", pipeline_run, img.name)
img.run.add(pipeline_run)
if pipeline_run not in skyreg.run.all():
- logger.info('Adding %s to sky region %s', pipeline_run, skyreg)
+ logger.info("Adding %s to sky region %s", pipeline_run, skyreg)
skyreg.run.add(pipeline_run)
def remove_duplicate_measurements(
- sources_df: pd.DataFrame,
- dup_lim: Optional[Angle] = None,
- ini_df: bool = False
+ sources_df: pd.DataFrame, dup_lim: Optional[Angle] = None, ini_df: bool = False
) -> pd.DataFrame:
"""
Remove perceived duplicate sources from a dataframe of loaded
@@ -240,87 +239,64 @@ def remove_duplicate_measurements(
Returns:
The input sources_df with duplicate sources removed.
"""
- logger.debug('Cleaning duplicate sources from epoch...')
+ logger.debug("Cleaning duplicate sources from epoch...")
if dup_lim is None:
dup_lim = Angle(2.5 * u.arcsec)
- logger.debug(
- 'Using duplicate crossmatch radius of %.2f arcsec.', dup_lim.arcsec
- )
+ logger.debug("Using duplicate crossmatch radius of %.2f arcsec.", dup_lim.arcsec)
# sort by the distance from the image centre so we know
# that the first source is always the one to keep
- sources_df = sources_df.sort_values(by='dist_from_centre')
+ sources_df = sources_df.sort_values(by="dist_from_centre")
- sources_sc = SkyCoord(
- sources_df['ra'],
- sources_df['dec'],
- unit=(u.deg, u.deg)
- )
+ sources_sc = SkyCoord(sources_df["ra"], sources_df["dec"], unit=(u.deg, u.deg))
# perform search around sky to get all self matches
- idxc, idxcatalog, *_ = sources_sc.search_around_sky(
- sources_sc, dup_lim
- )
+ idxc, idxcatalog, *_ = sources_sc.search_around_sky(sources_sc, dup_lim)
# create df from results
results = pd.DataFrame(
data={
- 'source_id': idxc,
- 'match_id': idxcatalog,
- 'source_image': sources_df.iloc[idxc]['image'].tolist(),
- 'match_image': sources_df.iloc[idxcatalog]['image'].tolist()
+ "source_id": idxc,
+ "match_id": idxcatalog,
+ "source_image": sources_df.iloc[idxc]["image"].tolist(),
+ "match_image": sources_df.iloc[idxcatalog]["image"].tolist(),
}
)
# Drop those that are matched from the same image
- matching_image_mask = (
- results['source_image'] != results['match_image']
- )
+ matching_image_mask = results["source_image"] != results["match_image"]
- results = (
- results.loc[matching_image_mask]
- .drop(['source_image', 'match_image'], axis=1)
+ results = results.loc[matching_image_mask].drop(
+ ["source_image", "match_image"], axis=1
)
# create a pair column defining each pair ith index
- results['pair'] = results.apply(tuple, 1).apply(sorted).apply(tuple)
+ results["pair"] = results.apply(tuple, 1).apply(sorted).apply(tuple)
# Drop the duplicate pairs (pairs are sorted so this works)
- results = results.drop_duplicates('pair')
+ results = results.drop_duplicates("pair")
# No longer need pair
- results = results.drop('pair', axis=1)
+ results = results.drop("pair", axis=1)
# Drop all self matches and we are left with those to drop
# in the match id column.
- to_drop = results.loc[
- results['source_id'] != results['match_id'],
- 'match_id'
- ]
+ to_drop = results.loc[results["source_id"] != results["match_id"], "match_id"]
# Get the index values from the ith values
to_drop_indexes = sources_df.iloc[to_drop].index.values
- logger.debug(
- "Dropping %i duplicate measurements.", to_drop_indexes.shape[0]
- )
+ logger.debug("Dropping %i duplicate measurements.", to_drop_indexes.shape[0])
# Drop them from sources
- sources_df = sources_df.drop(to_drop_indexes).sort_values(by='ra')
+ sources_df = sources_df.drop(to_drop_indexes).sort_values(by="ra")
# reset the source_df index
sources_df = sources_df.reset_index(drop=True)
- # Reset the source number
- if ini_df:
- sources_df['source'] = sources_df.index + 1
-
del results
return sources_df
def _load_measurements(
- image: Image,
- cols: List[str],
- start_id: int = 0,
- ini_df: bool = False
+ image: Image, cols: List[str], ini_df: bool = False
) -> pd.DataFrame:
"""
Load the measurements for an image from the parquet file.
@@ -331,9 +307,6 @@ def _load_measurements(
measurements.
cols:
The columns to load.
- start_id:
- The number to start from when setting the source ids (when
- 'ini_df' is 'True'). Defaults to 0.
ini_df:
Boolean to indicate whether these sources are part of the initial
source list creation for association. If 'True' the source ids are
@@ -343,27 +316,23 @@ def _load_measurements(
The measurements of the image with some extra values set ready for
association.
"""
- image_centre = SkyCoord(
- image.ra,
- image.dec,
- unit=(u.deg, u.deg)
- )
+ image_centre = SkyCoord(image.ra, image.dec, unit=(u.deg, u.deg))
df = pd.read_parquet(image.measurements_path, columns=cols)
- df['image'] = image.name
- df['datetime'] = image.datetime
+ df["image"] = image.name
+ df["datetime"] = image.datetime
# these are the first 'sources' if ini_df is True.
- df['source'] = df.index + start_id + 1 if ini_df else -1
- df['ra_source'] = df['ra']
- df['dec_source'] = df['dec']
- df['d2d'] = 0.
- df['dr'] = 0.
- df['related'] = None
+ df["source"] = df["id"].apply(lambda _: str(uuid.uuid4())) if ini_df else None
+ df["ra_source"] = df["ra"]
+ df["dec_source"] = df["dec"]
+ df["d2d"] = 0.0
+ df["dr"] = 0.0
+ df["related"] = None
- sources_sc = SkyCoord(df['ra'], df['dec'], unit=(u.deg, u.deg))
+ sources_sc = SkyCoord(df["ra"], df["dec"], unit=(u.deg, u.deg))
seps = sources_sc.separation(image_centre).degree
- df['dist_from_centre'] = seps
+ df["dist_from_centre"] = seps
del sources_sc
del seps
@@ -373,11 +342,11 @@ def _load_measurements(
def prep_skysrc_df(
images: List[Image],
- perc_error: float = 0.,
+ perc_error: float = 0.0,
duplicate_limit: Optional[Angle] = None,
- ini_df: bool = False
+ ini_df: bool = False,
) -> pd.DataFrame:
- '''
+ """
Initialise the source dataframe to use in association logic by
reading the measurement parquet file and creating columns. When epoch
based association is used it will also remove duplicate measurements from
@@ -402,25 +371,25 @@ def prep_skysrc_df(
Returns:
The measurements of the image(s) with some extra values set ready for
association and duplicates removed if necessary.
- '''
+ """
cols = [
- 'id',
- 'ra',
- 'uncertainty_ew',
- 'weight_ew',
- 'dec',
- 'uncertainty_ns',
- 'weight_ns',
- 'flux_int',
- 'flux_int_err',
- 'flux_int_isl_ratio',
- 'flux_peak',
- 'flux_peak_err',
- 'flux_peak_isl_ratio',
- 'forced',
- 'compactness',
- 'has_siblings',
- 'snr'
+ "id",
+ "ra",
+ "uncertainty_ew",
+ "weight_ew",
+ "dec",
+ "uncertainty_ns",
+ "weight_ns",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "forced",
+ "compactness",
+ "has_siblings",
+ "snr",
]
df = _load_measurements(images[0], cols, ini_df=ini_df)
@@ -428,34 +397,26 @@ def prep_skysrc_df(
if len(images) > 1:
for img in images[1:]:
df = pd.concat(
- [
- df,
- _load_measurements(
- img, cols, df.source.max(), ini_df=ini_df
- )
- ],
- ignore_index=True
+ [df, _load_measurements(img, cols, ini_df=ini_df)],
+ ignore_index=True,
)
- df = remove_duplicate_measurements(
- df, dup_lim=duplicate_limit, ini_df=ini_df
- )
+ df = remove_duplicate_measurements(df, dup_lim=duplicate_limit, ini_df=ini_df)
- df = df.drop('dist_from_centre', axis=1)
+ df = df.drop("dist_from_centre", axis=1)
if perc_error != 0.0:
- logger.info('Correcting flux errors with config error setting...')
- for col in ['flux_int', 'flux_peak']:
- df[f'{col}_err'] = np.hypot(
- df[f'{col}_err'].values, perc_error * df[col].values
+ logger.info("Correcting flux errors with config error setting...")
+ for col in ["flux_int", "flux_peak"]:
+ df[f"{col}_err"] = np.hypot(
+ df[f"{col}_err"].values, perc_error * df[col].values
)
return df
def add_new_one_to_many_relations(
- row: pd.Series, advanced: bool = False,
- source_ids: Optional[pd.DataFrame] = None
+ row: pd.Series, advanced: bool = False, source_ids: Optional[pd.DataFrame] = None
) -> List[int]:
"""
This handles the relation information being created from the
@@ -490,8 +451,8 @@ def add_new_one_to_many_relations(
if source_ids is None:
source_ids = pd.DataFrame()
- related_col = 'related_skyc1' if advanced else 'related'
- source_col = 'source_skyc1' if advanced else 'source'
+ related_col = "related_skyc1" if advanced else "related"
+ source_col = "source_skyc1" if advanced else "source"
# this is the not_original case where the original source id is appended.
if source_ids.empty:
@@ -499,7 +460,9 @@ def add_new_one_to_many_relations(
out = row[related_col]
out.append(row[source_col])
else:
- out = [row[source_col], ]
+ out = [
+ row[source_col],
+ ]
else: # the original case to append all the new ids.
source_ids = source_ids.loc[row[source_col]].iloc[0]
@@ -528,10 +491,10 @@ def add_new_many_to_one_relations(row: pd.Series) -> List[int]:
The new related field for the source in question, containing the
appended ids.
"""
- out = row['new_relations'].copy()
+ out = row["new_relations"].copy()
- if isinstance(row['related_skyc1'], list):
- out += row['related_skyc1'].copy()
+ if isinstance(row["related_skyc1"], list):
+ out += row["related_skyc1"].copy()
return out
@@ -547,17 +510,13 @@ def cross_join(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
Returns:
The resultant merged DataFrame.
"""
- return (
- left.assign(key=1)
- .merge(right.assign(key=1), on='key')
- .drop('key', axis=1)
- )
+ return left.assign(key=1).merge(right.assign(key=1), on="key").drop("key", axis=1)
def get_eta_metric(
row: Dict[str, float], df: pd.DataFrame, peak: bool = False
) -> float:
- '''
+ """
Calculates the eta variability metric of a source.
Works on the grouped by dataframe using the fluxes
of the associated measurements.
@@ -571,23 +530,22 @@ def get_eta_metric(
Returns:
The calculated eta value.
- '''
- if row['n_meas'] == 1:
- return 0.
-
- suffix = 'peak' if peak else 'int'
- weights = 1. / df[f'flux_{suffix}_err'].values**2
- fluxes = df[f'flux_{suffix}'].values
- eta = (row['n_meas'] / (row['n_meas']-1)) * (
- (weights * fluxes**2).mean() - (
- (weights * fluxes).mean()**2 / weights.mean()
- )
+ """
+ if row["n_meas"] == 1:
+ return 0.0
+
+ suffix = "peak" if peak else "int"
+ weights = 1.0 / df[f"flux_{suffix}_err"].values ** 2
+ fluxes = df[f"flux_{suffix}"].values
+ eta = (row["n_meas"] / (row["n_meas"] - 1)) * (
+ (weights * fluxes**2).mean()
+ - ((weights * fluxes).mean() ** 2 / weights.mean())
)
return eta
def groupby_funcs(df: pd.DataFrame) -> pd.Series:
- '''
+ """
Performs calculations on the unique sources to get the
lightcurve properties. Works on the grouped by source
dataframe.
@@ -598,69 +556,61 @@ def groupby_funcs(df: pd.DataFrame) -> pd.Series:
Returns:
Pandas series containing the calculated metrics of the source.
- '''
+ """
# calculated average ra, dec, fluxes and metrics
d = {}
- d['img_list'] = df['image'].values.tolist()
- d['n_meas_forced'] = df['forced'].sum()
- d['n_meas'] = df['id'].count()
- d['n_meas_sel'] = d['n_meas'] - d['n_meas_forced']
- d['n_sibl'] = df['has_siblings'].sum()
- if d['n_meas_forced'] > 0:
- non_forced_sel = ~df['forced']
- d['wavg_ra'] = (
- df.loc[non_forced_sel, 'interim_ew'].sum() /
- df.loc[non_forced_sel, 'weight_ew'].sum()
+ d["img_list"] = df["image"].values.tolist()
+ d["n_meas_forced"] = df["forced"].sum()
+ d["n_meas"] = df["id"].count()
+ d["n_meas_sel"] = d["n_meas"] - d["n_meas_forced"]
+ d["n_sibl"] = df["has_siblings"].sum()
+ if d["n_meas_forced"] > 0:
+ non_forced_sel = ~df["forced"]
+ d["wavg_ra"] = (
+ df.loc[non_forced_sel, "interim_ew"].sum()
+ / df.loc[non_forced_sel, "weight_ew"].sum()
)
- d['wavg_dec'] = (
- df.loc[non_forced_sel, 'interim_ns'].sum() /
- df.loc[non_forced_sel, 'weight_ns'].sum()
+ d["wavg_dec"] = (
+ df.loc[non_forced_sel, "interim_ns"].sum()
+ / df.loc[non_forced_sel, "weight_ns"].sum()
)
- d['avg_compactness'] = df.loc[
- non_forced_sel, 'compactness'
- ].mean()
- d['min_snr'] = df.loc[
- non_forced_sel, 'snr'
- ].min()
- d['max_snr'] = df.loc[
- non_forced_sel, 'snr'
- ].max()
+ d["avg_compactness"] = df.loc[non_forced_sel, "compactness"].mean()
+ d["min_snr"] = df.loc[non_forced_sel, "snr"].min()
+ d["max_snr"] = df.loc[non_forced_sel, "snr"].max()
else:
- d['wavg_ra'] = df['interim_ew'].sum() / df['weight_ew'].sum()
- d['wavg_dec'] = df['interim_ns'].sum() / df['weight_ns'].sum()
- d['avg_compactness'] = df['compactness'].mean()
- d['min_snr'] = df['snr'].min()
- d['max_snr'] = df['snr'].max()
-
- d['wavg_uncertainty_ew'] = 1. / np.sqrt(df['weight_ew'].sum())
- d['wavg_uncertainty_ns'] = 1. / np.sqrt(df['weight_ns'].sum())
- for col in ['avg_flux_int', 'avg_flux_peak']:
- d[col] = df[col.split('_', 1)[1]].mean()
- for col in ['max_flux_peak', 'max_flux_int']:
- d[col] = df[col.split('_', 1)[1]].max()
- for col in ['min_flux_peak', 'min_flux_int']:
- d[col] = df[col.split('_', 1)[1]].min()
- for col in ['min_flux_peak_isl_ratio', 'min_flux_int_isl_ratio']:
- d[col] = df[col.split('_', 1)[1]].min()
-
- for col in ['flux_int', 'flux_peak']:
- d[f'{col}_sq'] = (df[col]**2).mean()
- d['v_int'] = df['flux_int'].std() / df['flux_int'].mean()
- d['v_peak'] = df['flux_peak'].std() / df['flux_peak'].mean()
- d['eta_int'] = get_eta_metric(d, df)
- d['eta_peak'] = get_eta_metric(d, df, peak=True)
+ d["wavg_ra"] = df["interim_ew"].sum() / df["weight_ew"].sum()
+ d["wavg_dec"] = df["interim_ns"].sum() / df["weight_ns"].sum()
+ d["avg_compactness"] = df["compactness"].mean()
+ d["min_snr"] = df["snr"].min()
+ d["max_snr"] = df["snr"].max()
+
+ d["wavg_uncertainty_ew"] = 1.0 / np.sqrt(df["weight_ew"].sum())
+ d["wavg_uncertainty_ns"] = 1.0 / np.sqrt(df["weight_ns"].sum())
+ for col in ["avg_flux_int", "avg_flux_peak"]:
+ d[col] = df[col.split("_", 1)[1]].mean()
+ for col in ["max_flux_peak", "max_flux_int"]:
+ d[col] = df[col.split("_", 1)[1]].max()
+ for col in ["min_flux_peak", "min_flux_int"]:
+ d[col] = df[col.split("_", 1)[1]].min()
+ for col in ["min_flux_peak_isl_ratio", "min_flux_int_isl_ratio"]:
+ d[col] = df[col.split("_", 1)[1]].min()
+
+ for col in ["flux_int", "flux_peak"]:
+ d[f"{col}_sq"] = (df[col] ** 2).mean()
+ d["v_int"] = df["flux_int"].std() / df["flux_int"].mean()
+ d["v_peak"] = df["flux_peak"].std() / df["flux_peak"].mean()
+ d["eta_int"] = get_eta_metric(d, df)
+ d["eta_peak"] = get_eta_metric(d, df, peak=True)
# remove not used cols
- for col in ['flux_int_sq', 'flux_peak_sq']:
+ for col in ["flux_int_sq", "flux_peak_sq"]:
d.pop(col)
# get unique related sources
- list_uniq_related = list(set(
- chain.from_iterable(
- lst for lst in df['related'] if isinstance(lst, list)
- )
- ))
- d['related_list'] = list_uniq_related if list_uniq_related else -1
+ list_uniq_related = list(
+ set(chain.from_iterable(lst for lst in df["related"] if isinstance(lst, list)))
+ )
+ d["related_list"] = list_uniq_related if list_uniq_related else -1
return pd.Series(d).fillna(value={"v_int": 0.0, "v_peak": 0.0})
@@ -677,44 +627,41 @@ def parallel_groupby(df: pd.DataFrame) -> pd.DataFrame:
The source dataframe with the calculated metric columns.
"""
col_dtype = {
- 'img_list': 'O',
- 'n_meas_forced': 'i',
- 'n_meas': 'i',
- 'n_meas_sel': 'i',
- 'n_sibl': 'i',
- 'wavg_ra': 'f',
- 'wavg_dec': 'f',
- 'avg_compactness': 'f',
- 'min_snr': 'f',
- 'max_snr': 'f',
- 'wavg_uncertainty_ew': 'f',
- 'wavg_uncertainty_ns': 'f',
- 'avg_flux_int': 'f',
- 'avg_flux_peak': 'f',
- 'max_flux_peak': 'f',
- 'max_flux_int': 'f',
- 'min_flux_peak': 'f',
- 'min_flux_int': 'f',
- 'min_flux_peak_isl_ratio': 'f',
- 'min_flux_int_isl_ratio': 'f',
- 'v_int': 'f',
- 'v_peak': 'f',
- 'eta_int': 'f',
- 'eta_peak': 'f',
- 'related_list': 'O'
+ "img_list": "O",
+ "n_meas_forced": "i",
+ "n_meas": "i",
+ "n_meas_sel": "i",
+ "n_sibl": "i",
+ "wavg_ra": "f",
+ "wavg_dec": "f",
+ "avg_compactness": "f",
+ "min_snr": "f",
+ "max_snr": "f",
+ "wavg_uncertainty_ew": "f",
+ "wavg_uncertainty_ns": "f",
+ "avg_flux_int": "f",
+ "avg_flux_peak": "f",
+ "max_flux_peak": "f",
+ "max_flux_int": "f",
+ "min_flux_peak": "f",
+ "min_flux_int": "f",
+ "min_flux_peak_isl_ratio": "f",
+ "min_flux_int_isl_ratio": "f",
+ "v_int": "f",
+ "v_peak": "f",
+ "eta_int": "f",
+ "eta_peak": "f",
+ "related_list": "O",
}
n_cpu = cpu_count() - 1
out = dd.from_pandas(df, n_cpu)
out = (
- out.groupby('source')
- .apply(
- groupby_funcs,
- meta=col_dtype
- )
- .compute(num_workers=n_cpu, scheduler='processes')
+ out.groupby("source")
+ .apply(groupby_funcs, meta=col_dtype)
+ .compute(num_workers=n_cpu, scheduler="processes")
)
- out['n_rel'] = out['related_list'].apply(lambda x: 0 if x == -1 else len(x))
+ out["n_rel"] = out["related_list"].apply(lambda x: 0 if x == -1 else len(x))
return out
@@ -734,11 +681,11 @@ def calc_ave_coord(grp: pd.DataFrame) -> pd.Series:
image and epoch lists.
"""
d = {}
- grp = grp.sort_values(by='datetime')
- d['img_list'] = grp['image'].values.tolist()
- d['epoch_list'] = grp['epoch'].values.tolist()
- d['wavg_ra'] = grp['interim_ew'].sum() / grp['weight_ew'].sum()
- d['wavg_dec'] = grp['interim_ns'].sum() / grp['weight_ns'].sum()
+ grp = grp.sort_values(by="datetime")
+ d["img_list"] = grp["image"].values.tolist()
+ d["epoch_list"] = grp["epoch"].values.tolist()
+ d["wavg_ra"] = grp["interim_ew"].sum() / grp["weight_ew"].sum()
+ d["wavg_dec"] = grp["interim_ns"].sum() / grp["weight_ns"].sum()
return pd.Series(d)
@@ -757,24 +704,24 @@ def parallel_groupby_coord(df: pd.DataFrame) -> pd.DataFrame:
lists for each unique source (group).
"""
col_dtype = {
- 'img_list': 'O',
- 'epoch_list': 'O',
- 'wavg_ra': 'f',
- 'wavg_dec': 'f',
+ "img_list": "O",
+ "epoch_list": "O",
+ "wavg_ra": "f",
+ "wavg_dec": "f",
}
n_cpu = cpu_count() - 1
out = dd.from_pandas(df, n_cpu)
out = (
- out.groupby('source')
+ out.groupby("source")
.apply(calc_ave_coord, meta=col_dtype)
- .compute(num_workers=n_cpu, scheduler='processes')
+ .compute(num_workers=n_cpu, scheduler="processes")
)
return out
def get_rms_noise_image_values(rms_path: str) -> Tuple[float, float, float]:
- '''
+ """
Open the RMS noise FITS file and compute the median, max and min
rms values to be added to the image model and then used in the
calculations.
@@ -789,20 +736,20 @@ def get_rms_noise_image_values(rms_path: str) -> Tuple[float, float, float]:
Raises:
IOError: Raised when the RMS FITS file cannot be found.
- '''
- logger.debug('Extracting Image RMS values from Noise file...')
- med_val = min_val = max_val = 0.
+ """
+ logger.debug("Extracting Image RMS values from Noise file...")
+ med_val = min_val = max_val = 0.0
try:
with open_fits(rms_path) as f:
data = f[0].data
data = data[np.logical_not(np.isnan(data))]
data = data[data != 0]
- med_val = np.median(data) * 1e+3
- min_val = np.min(data) * 1e+3
- max_val = np.max(data) * 1e+3
+ med_val = np.median(data) * 1e3
+ min_val = np.min(data) * 1e3
+ max_val = np.max(data) * 1e3
del data
except Exception:
- raise IOError(f'Could not read this RMS FITS file: {rms_path}')
+ raise IOError(f"Could not read this RMS FITS file: {rms_path}")
return med_val, min_val, max_val
@@ -821,9 +768,7 @@ def get_image_list_diff(row: pd.Series) -> Union[List[str], int]:
A list of the images missing from the observed image list.
A '-1' integer value if there are no missing images.
"""
- out = list(
- filter(lambda arg: arg not in row['img_list'], row['skyreg_img_list'])
- )
+ out = list(filter(lambda arg: arg not in row["img_list"], row["skyreg_img_list"]))
# set empty list to -1
if not out:
@@ -832,15 +777,15 @@ def get_image_list_diff(row: pd.Series) -> Union[List[str], int]:
# Check that an epoch has not already been seen (just not in the 'ideal'
# image)
out_epochs = [
- row['skyreg_epoch'][pair[0]] for pair in enumerate(
- row['skyreg_img_list']
- ) if pair[1] in out
+ row["skyreg_epoch"][pair[0]]
+ for pair in enumerate(row["skyreg_img_list"])
+ if pair[1] in out
]
out = [
- out[pair[0]] for pair in enumerate(
- out_epochs
- ) if pair[1] not in row['epoch_list']
+ out[pair[0]]
+ for pair in enumerate(out_epochs)
+ if pair[1] not in row["epoch_list"]
]
if not out:
@@ -863,11 +808,20 @@ def get_names_and_epochs(grp: pd.DataFrame) -> pd.Series:
image names, epochs and datetimes.
"""
d = {}
- d['skyreg_img_epoch_list'] = [[[x, ], y, z] for x, y, z in zip(
- grp['name'].values.tolist(),
- grp['epoch'].values.tolist(),
- grp['datetime'].values.tolist()
- )]
+ d["skyreg_img_epoch_list"] = [
+ [
+ [
+ x,
+ ],
+ y,
+ z,
+ ]
+ for x, y, z in zip(
+ grp["name"].values.tolist(),
+ grp["epoch"].values.tolist(),
+ grp["datetime"].values.tolist(),
+ )
+ ]
return pd.Series(d)
@@ -884,7 +838,7 @@ def check_primary_image(row: pd.Series) -> bool:
Returns:
True if primary in image list else False.
"""
- return row['primary'] in row['img_list']
+ return row["primary"] in row["img_list"]
def get_src_skyregion_merged_df(
@@ -951,32 +905,22 @@ def get_src_skyregion_merged_df(
merged_timer = StopWatch()
- skyreg_df = skyreg_df.drop(
- ['x', 'y', 'z', 'width_ra', 'width_dec'], axis=1
- )
+ skyreg_df = skyreg_df.drop(["x", "y", "z", "width_ra", "width_dec"], axis=1)
- images_df['name'] = images_df['image_dj'].apply(
- lambda x: x.name
- )
- images_df['datetime'] = images_df['image_dj'].apply(
- lambda x: x.datetime
- )
+ images_df["name"] = images_df["image_dj"].apply(lambda x: x.name)
+ images_df["datetime"] = images_df["image_dj"].apply(lambda x: x.datetime)
skyreg_df = skyreg_df.join(
- pd.DataFrame(
- images_df.groupby('skyreg_id').apply(
- get_names_and_epochs
- )
- ),
- on='id'
+ pd.DataFrame(images_df.groupby("skyreg_id").apply(get_names_and_epochs)),
+ on="id",
)
- sources_df = sources_df.sort_values(by='datetime')
+ sources_df = sources_df.sort_values(by="datetime")
# calculate some metrics on sources
# compute only some necessary metrics in the groupby
timer = StopWatch()
srcs_df = parallel_groupby_coord(sources_df)
- logger.debug('Groupby-apply time: %.2f seconds', timer.reset())
+ logger.debug("Groupby-apply time: %.2f seconds", timer.reset())
del sources_df
@@ -1009,89 +953,60 @@ def get_src_skyregion_merged_df(
del skyreg_df
- src_skyrg_df[
- ['skyreg_img_list', 'skyreg_epoch', 'skyreg_datetime']
- ] = pd.DataFrame(
- src_skyrg_df['skyreg_img_epoch_list'].tolist(),
- index=src_skyrg_df.index
+ src_skyrg_df[["skyreg_img_list", "skyreg_epoch", "skyreg_datetime"]] = pd.DataFrame(
+ src_skyrg_df["skyreg_img_epoch_list"].tolist(), index=src_skyrg_df.index
)
- src_skyrg_df = src_skyrg_df.drop('skyreg_img_epoch_list', axis=1)
+ src_skyrg_df = src_skyrg_df.drop("skyreg_img_epoch_list", axis=1)
src_skyrg_df = (
- src_skyrg_df.sort_values(
- ['source', 'sep']
- )
- .drop_duplicates(['source', 'skyreg_epoch'])
- .sort_values(by='skyreg_datetime')
- .drop(
- ['sep', 'skyreg_datetime'],
- axis=1
- )
+ src_skyrg_df.sort_values(["source", "sep"])
+ .drop_duplicates(["source", "skyreg_epoch"])
+ .sort_values(by="skyreg_datetime")
+ .drop(["sep", "skyreg_datetime"], axis=1)
)
# annoyingly epoch needs to be not a list to drop duplicates
# but then we need to sum the epochs into a list
- src_skyrg_df['skyreg_epoch'] = src_skyrg_df['skyreg_epoch'].apply(
- lambda x: [x, ]
+ src_skyrg_df["skyreg_epoch"] = src_skyrg_df["skyreg_epoch"].apply(
+ lambda x: [
+ x,
+ ]
)
- src_skyrg_df = (
- src_skyrg_df.groupby('source')
- .sum(numeric_only=False) # sum because we need to preserve order
- )
+ src_skyrg_df = src_skyrg_df.groupby("source").sum(
+ numeric_only=False
+ ) # sum because we need to preserve order
# merge into main df and compare the images
- srcs_df = srcs_df.merge(
- src_skyrg_df, left_index=True, right_index=True
- )
+ srcs_df = srcs_df.merge(src_skyrg_df, left_index=True, right_index=True)
del src_skyrg_df
- srcs_df['img_diff'] = srcs_df[
- ['img_list', 'skyreg_img_list', 'epoch_list', 'skyreg_epoch']
- ].apply(
- get_image_list_diff, axis=1
- )
+ srcs_df["img_diff"] = srcs_df[
+ ["img_list", "skyreg_img_list", "epoch_list", "skyreg_epoch"]
+ ].apply(get_image_list_diff, axis=1)
- srcs_df = srcs_df.loc[
- srcs_df['img_diff'] != -1
- ]
+ srcs_df = srcs_df.loc[srcs_df["img_diff"] != -1]
- srcs_df = srcs_df.drop(
- ['epoch_list', 'skyreg_epoch'],
- axis=1
- )
+ srcs_df = srcs_df.drop(["epoch_list", "skyreg_epoch"], axis=1)
- srcs_df['primary'] = srcs_df[
- 'skyreg_img_list'
- ].apply(lambda x: x[0])
+ srcs_df["primary"] = srcs_df["skyreg_img_list"].apply(lambda x: x[0])
- srcs_df['detection'] = srcs_df[
- 'img_list'
- ].apply(lambda x: x[0])
+ srcs_df["detection"] = srcs_df["img_list"].apply(lambda x: x[0])
- srcs_df['in_primary'] = srcs_df[
- ['primary', 'img_list']
- ].apply(
- check_primary_image,
- axis=1
+ srcs_df["in_primary"] = srcs_df[["primary", "img_list"]].apply(
+ check_primary_image, axis=1
)
- srcs_df = srcs_df.drop(['img_list', 'skyreg_img_list', 'primary'], axis=1)
+ srcs_df = srcs_df.drop(["img_list", "skyreg_img_list", "primary"], axis=1)
- logger.info(
- 'Ideal source coverage time: %.2f seconds', merged_timer.reset()
- )
+ logger.info("Ideal source coverage time: %.2f seconds", merged_timer.reset())
return srcs_df
-def _get_skyregion_relations(
- row: pd.Series,
- coords: SkyCoord,
- ids: pd.core.indexes.numeric.Int64Index
-) -> List[int]:
- '''
+def _get_skyregion_relations(row: pd.Series, coords: SkyCoord, ids: int) -> List[int]:
+ """
For each sky region row a list is returned that
contains the ids of other sky regions that overlap
with the row sky region (including itself).
@@ -1107,18 +1022,14 @@ def _get_skyregion_relations(
Returns:
A list of other sky regions (including self) that are within the
'xtr_radius' of the sky region in the row.
- '''
- target = SkyCoord(
- row['centre_ra'],
- row['centre_dec'],
- unit=(u.deg, u.deg)
- )
+ """
+ target = SkyCoord(row["centre_ra"], row["centre_dec"], unit=(u.deg, u.deg))
seps = target.separation(coords)
# place a slight buffer on the radius to make sure
# any neighbouring fields are caught
- mask = seps <= row['xtr_radius'] * 1.1 * u.deg
+ mask = seps <= row["xtr_radius"] * 1.1 * u.deg
related_ids = ids[mask].to_list()
@@ -1155,28 +1066,19 @@ def group_skyregions(df: pd.DataFrame) -> pd.DataFrame:
| 1 | 2 |
+----+----------------+
"""
- sr_coords = SkyCoord(
- df['centre_ra'],
- df['centre_dec'],
- unit=(u.deg, u.deg)
- )
+ sr_coords = SkyCoord(df["centre_ra"], df["centre_dec"], unit=(u.deg, u.deg))
- df = df.set_index('id')
+ df = df.set_index("id")
- results = df.apply(
- _get_skyregion_relations,
- args=(sr_coords, df.index),
- axis=1
- )
+ results = df.apply(_get_skyregion_relations, args=(sr_coords, df.index), axis=1)
skyreg_groups: Dict[int, List[Any]] = {}
master_done = [] # keep track of all checked ids in master done
- for skyreg_id, neighbours in results.iteritems():
-
+ for skyreg_id, neighbours in results.items():
if skyreg_id not in master_done:
- local_done = [] # a local done list for the sky region group.
+ local_done = [] # a local done list for the sky region group.
# add the current skyreg_id to both master and local done.
master_done.append(skyreg_id)
local_done.append(skyreg_id)
@@ -1221,9 +1123,9 @@ def group_skyregions(df: pd.DataFrame) -> pd.DataFrame:
for j in skyreg_groups[i]:
skyreg_group_ids[j] = i
- skyreg_group_ids = pd.DataFrame.from_dict(
- skyreg_group_ids, orient='index'
- ).rename(columns={0: 'skyreg_group'})
+ skyreg_group_ids = pd.DataFrame.from_dict(skyreg_group_ids, orient="index").rename(
+ columns={0: "skyreg_group"}
+ )
return skyreg_group_ids
@@ -1264,27 +1166,22 @@ def get_parallel_assoc_image_df(
# | 6 | VAST_2118-06A.EPOCH06x.I.fits | 3 | 1 |
# | 7 | VAST_0127-73A.EPOCH08.I.fits | 1 | 2 |
# +----+-------------------------------+-------------+----------------+
- skyreg_ids = [i.skyreg_id for i in images]
+ skyreg_ids = [str(i.skyreg_id) for i in images]
- images_df = pd.DataFrame({
- 'image_dj': images,
- 'skyreg_id': skyreg_ids,
- })
+ images_df = pd.DataFrame(
+ {
+ "image_dj": images,
+ "skyreg_id": skyreg_ids,
+ }
+ )
images_df = images_df.merge(
- skyregion_groups,
- how='left',
- left_on='skyreg_id',
- right_index=True
+ skyregion_groups, how="left", left_on="skyreg_id", right_index=True
)
- images_df['image_name'] = images_df['image_dj'].apply(
- lambda x: x.name
- )
+ images_df["image_name"] = images_df["image_dj"].apply(lambda x: x.name)
- images_df['image_datetime'] = images_df['image_dj'].apply(
- lambda x: x.datetime
- )
+ images_df["image_datetime"] = images_df["image_dj"].apply(lambda x: x.datetime)
return images_df
@@ -1301,58 +1198,41 @@ def create_measurements_arrow_file(p_run: Run) -> None:
Returns:
None
"""
- logger.info('Creating measurements.arrow for run %s.', p_run.name)
+ logger.info("Creating measurements.arrow for run %s.", p_run.name)
- associations = pd.read_parquet(
- os.path.join(
- p_run.path,
- 'associations.parquet'
- )
- )
- images = pd.read_parquet(
- os.path.join(
- p_run.path,
- 'images.parquet'
- )
- )
+ associations = pd.read_parquet(os.path.join(p_run.path, "associations.parquet"))
+ images = pd.read_parquet(os.path.join(p_run.path, "images.parquet"))
- m_files = images['measurements_path'].tolist()
+ m_files = images["measurements_path"].tolist()
- m_files += glob.glob(os.path.join(
- p_run.path,
- 'forced*.parquet'
- ))
+ m_files += glob.glob(os.path.join(p_run.path, "forced*.parquet"))
- logger.debug('Loading %i files...', len(m_files))
- measurements = dd.read_parquet(m_files, engine='pyarrow').compute()
+ logger.debug("Loading %i files...", len(m_files))
+ measurements = dd.read_parquet(m_files, engine="pyarrow").compute()
measurements = measurements.loc[
- measurements['id'].isin(associations['meas_id'].values)
+ measurements["id"].isin(associations["meas_id"].values)
]
measurements = (
- associations.loc[:, ['meas_id', 'source_id']]
- .set_index('meas_id')
- .merge(
- measurements,
- left_index=True,
- right_on='id'
- )
- .rename(columns={'source_id': 'source'})
+ associations.loc[:, ["meas_id", "source_id"]]
+ .set_index("meas_id")
+ .merge(measurements, left_index=True, right_on="id")
+ .rename(columns={"source_id": "source"})
)
# drop timezone from datetime for vaex compatibility
# TODO: Look to keep the timezone if/when vaex is compatible.
- measurements['time'] = measurements['time'].dt.tz_localize(None)
+ measurements["time"] = measurements["time"].dt.tz_localize(None)
- logger.debug('Optimising dataframes.')
+ logger.debug("Optimising dataframes.")
measurements = optimize_ints(optimize_floats(measurements))
logger.debug("Loading to pyarrow table.")
measurements = pa.Table.from_pandas(measurements)
logger.debug("Exporting to arrow file.")
- outname = os.path.join(p_run.path, 'measurements.arrow')
+ outname = os.path.join(p_run.path, "measurements.arrow")
local = pa.fs.LocalFileSystem()
@@ -1373,23 +1253,20 @@ def create_measurement_pairs_arrow_file(p_run: Run) -> None:
Returns:
None
"""
- logger.info('Creating measurement_pairs.arrow for run %s.', p_run.name)
+ logger.info("Creating measurement_pairs.arrow for run %s.", p_run.name)
measurement_pairs_df = pd.read_parquet(
- os.path.join(
- p_run.path,
- 'measurement_pairs.parquet'
- )
+ os.path.join(p_run.path, "measurement_pairs.parquet")
)
- logger.debug('Optimising dataframe.')
+ logger.debug("Optimising dataframe.")
measurement_pairs_df = optimize_ints(optimize_floats(measurement_pairs_df))
logger.debug("Loading to pyarrow table.")
measurement_pairs_df = pa.Table.from_pandas(measurement_pairs_df)
logger.debug("Exporting to arrow file.")
- outname = os.path.join(p_run.path, 'measurement_pairs.arrow')
+ outname = os.path.join(p_run.path, "measurement_pairs.arrow")
local = pa.fs.LocalFileSystem()
@@ -1413,12 +1290,13 @@ def backup_parquets(p_run_path: str) -> None:
parquets = (
glob.glob(os.path.join(p_run_path, "*.parquet"))
# TODO Remove arrow when arrow files are no longer required.
- + glob.glob(os.path.join(p_run_path, "*.arrow")))
+ + glob.glob(os.path.join(p_run_path, "*.arrow"))
+ )
for i in parquets:
- backup_name = i + '.bak'
+ backup_name = i + ".bak"
if os.path.isfile(backup_name):
- logger.debug(f'Removing old backup file: {backup_name}.')
+ logger.debug(f"Removing old backup file: {backup_name}.")
os.remove(backup_name)
shutil.copyfile(i, backup_name)
@@ -1437,18 +1315,17 @@ def create_temp_config_file(p_run_path: str) -> None:
Returns:
None
"""
- config_name = 'config.yaml'
- temp_config_name = 'config_temp.yaml'
+ config_name = "config.yaml"
+ temp_config_name = "config_temp.yaml"
shutil.copyfile(
os.path.join(p_run_path, config_name),
- os.path.join(p_run_path, temp_config_name)
+ os.path.join(p_run_path, temp_config_name),
)
def reconstruct_associtaion_dfs(
- images_df_done: pd.DataFrame,
- previous_parquet_paths: Dict[str, str]
+ images_df_done: pd.DataFrame, previous_parquet_paths: Dict[str, str]
) -> Tuple[pd.DataFrame, pd.DataFrame]:
"""
This function is used with add image mode and performs the necessary
@@ -1467,27 +1344,22 @@ def reconstruct_associtaion_dfs(
The reconstructed `sources_df` dataframe.
The reconstructed `skyc1_srs` dataframes.
"""
- prev_associations = pd.read_parquet(previous_parquet_paths['associations'])
+ prev_associations = pd.read_parquet(previous_parquet_paths["associations"])
# Get the parquet paths from the image objects
img_meas_paths = (
- images_df_done['image_dj'].apply(lambda x: x.measurements_path)
- .to_list()
+ images_df_done["image_dj"].apply(lambda x: x.measurements_path).to_list()
)
# Obtain the pipeline run path in order to fetch forced measurements.
- run_path = previous_parquet_paths['sources'].replace(
- 'sources.parquet.bak', ''
- )
+ run_path = previous_parquet_paths["sources"].replace("sources.parquet.bak", "")
# Get the forced measurement paths.
img_fmeas_paths = []
for i in images_df_done.image_name.values:
forced_parquet = os.path.join(
- run_path, "forced_measurements_{}.parquet".format(
- i.replace(".", "_")
- )
+ run_path, "forced_measurements_{}.parquet".format(i.replace(".", "_"))
)
if os.path.isfile(forced_parquet):
img_fmeas_paths.append(forced_parquet)
@@ -1497,178 +1369,229 @@ def reconstruct_associtaion_dfs(
# Define the columns that are required
cols = [
- 'id',
- 'ra',
- 'uncertainty_ew',
- 'weight_ew',
- 'dec',
- 'uncertainty_ns',
- 'weight_ns',
- 'flux_int',
- 'flux_int_err',
- 'flux_int_isl_ratio',
- 'flux_peak',
- 'flux_peak_err',
- 'flux_peak_isl_ratio',
- 'forced',
- 'compactness',
- 'has_siblings',
- 'snr',
- 'image_id',
- 'time',
+ "id",
+ "ra",
+ "uncertainty_ew",
+ "weight_ew",
+ "dec",
+ "uncertainty_ns",
+ "weight_ns",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "forced",
+ "compactness",
+ "has_siblings",
+ "snr",
+ "image_id",
+ "time",
]
# Open all the parquets
- logger.debug(
- "Opening all measurement parquet files to use in reconstruction..."
- )
- measurements = pd.concat(
- [pd.read_parquet(f, columns=cols) for f in img_meas_paths]
- )
+ logger.debug("Opening all measurement parquet files to use in reconstruction...")
+ measurements = pd.concat([pd.read_parquet(f, columns=cols) for f in img_meas_paths])
# Create mask to drop measurements for epoch mode (epoch based mode).
- measurements_mask = measurements['id'].isin(
- prev_associations['meas_id'])
- measurements = measurements.loc[measurements_mask].set_index('id')
+ measurements_mask = measurements["id"].isin(prev_associations["meas_id"])
+ measurements = measurements.loc[measurements_mask].set_index("id")
# Set the index on images_df for faster merging.
- images_df_done['image_id'] = images_df_done['image_dj'].apply(
- lambda x: x.id).values
- images_df_done = images_df_done.set_index('image_id')
+ images_df_done["image_id"] = images_df_done["image_dj"].apply(lambda x: str(x.id)).values
+ images_df_done = images_df_done.set_index("image_id")
# Merge image information to measurements
- measurements = (
- measurements.merge(
- images_df_done[['image_name', 'epoch']],
- left_on='image_id', right_index=True
- )
- .rename(columns={'image_name': 'image'})
- )
+ measurements = measurements.merge(
+ images_df_done[["image_name", "epoch"]], left_on="image_id", right_index=True
+ ).rename(columns={"image_name": "image"})
# Drop any associations that are not used in this sky region group.
- associations_mask = prev_associations['meas_id'].isin(
- measurements.index.values)
+ associations_mask = prev_associations["meas_id"].isin(measurements.index.values)
prev_associations = prev_associations.loc[associations_mask]
# Merge measurements into the associations to form the sources_df.
- sources_df = (
- prev_associations.merge(
- measurements, left_on='meas_id', right_index=True
- )
- .rename(columns={
- 'source_id': 'source', 'time': 'datetime', 'meas_id': 'id',
- 'ra': 'ra_source', 'dec': 'dec_source',
- 'uncertainty_ew': 'uncertainty_ew_source',
- 'uncertainty_ns': 'uncertainty_ns_source',
- })
+ sources_df = prev_associations.merge(
+ measurements, left_on="meas_id", right_index=True
+ ).rename(
+ columns={
+ "source_id": "source",
+ "time": "datetime",
+ "meas_id": "id",
+ "ra": "ra_source",
+ "dec": "dec_source",
+ "uncertainty_ew": "uncertainty_ew_source",
+ "uncertainty_ns": "uncertainty_ns_source",
+ }
)
# Load up the previous unique sources.
prev_sources = pd.read_parquet(
- previous_parquet_paths['sources'], columns=[
- 'wavg_ra', 'wavg_dec',
- 'wavg_uncertainty_ew', 'wavg_uncertainty_ns',
- ]
+ previous_parquet_paths["sources"],
+ columns=[
+ "wavg_ra",
+ "wavg_dec",
+ "wavg_uncertainty_ew",
+ "wavg_uncertainty_ns",
+ ],
)
# Merge the wavg ra and dec to the sources_df - this is required to
# create the skyc1_srcs below (but MUST be converted back to the source
# ra and dec)
- sources_df = (
- sources_df.merge(
- prev_sources, left_on='source', right_index=True)
- .rename(columns={
- 'wavg_ra': 'ra', 'wavg_dec': 'dec',
- 'wavg_uncertainty_ew': 'uncertainty_ew',
- 'wavg_uncertainty_ns': 'uncertainty_ns',
- })
+ sources_df = sources_df.merge(
+ prev_sources, left_on="source", right_index=True
+ ).rename(
+ columns={
+ "wavg_ra": "ra",
+ "wavg_dec": "dec",
+ "wavg_uncertainty_ew": "uncertainty_ew",
+ "wavg_uncertainty_ns": "uncertainty_ns",
+ }
)
# Load the previous relations
- prev_relations = pd.read_parquet(previous_parquet_paths['relations'])
+ prev_relations = pd.read_parquet(previous_parquet_paths["relations"])
# Form relation lists to merge in.
prev_relations = pd.DataFrame(
- prev_relations
- .groupby('from_source_id')['to_source_id']
- .apply(lambda x: x.values.tolist())
- ).rename(columns={'to_source_id': 'related'})
+ prev_relations.groupby("from_source_id")["to_source_id"].apply(
+ lambda x: x.values.tolist()
+ )
+ ).rename(columns={"to_source_id": "related"})
# Append the relations to only the last instance of each source
# First get the ids of the sources
- relation_ids = sources_df[
- sources_df.source.isin(prev_relations.index.values)].drop_duplicates(
- 'source', keep='last'
- ).index.values
+ relation_ids = (
+ sources_df[sources_df.source.isin(prev_relations.index.values)]
+ .drop_duplicates("source", keep="last")
+ .index.values
+ )
# Make sure we attach the correct source id
source_ids = sources_df.loc[relation_ids].source.values
- sources_df['related'] = np.nan
+ sources_df["related"] = np.nan
relations_to_update = prev_relations.loc[source_ids].to_numpy().copy()
- relations_to_update = np.reshape(
- relations_to_update, relations_to_update.shape[0])
- sources_df.loc[relation_ids, 'related'] = relations_to_update
+ relations_to_update = np.reshape(relations_to_update, relations_to_update.shape[0])
+ sources_df.loc[relation_ids, "related"] = relations_to_update
# Reorder so we don't mess up the dask metas.
- sources_df = sources_df[[
- 'id', 'uncertainty_ew', 'weight_ew', 'uncertainty_ns', 'weight_ns',
- 'flux_int', 'flux_int_err', 'flux_int_isl_ratio', 'flux_peak',
- 'flux_peak_err', 'flux_peak_isl_ratio', 'forced', 'compactness',
- 'has_siblings', 'snr', 'image', 'datetime', 'source', 'ra', 'dec',
- 'ra_source', 'dec_source', 'd2d', 'dr', 'related', 'epoch',
- 'uncertainty_ew_source', 'uncertainty_ns_source'
- ]]
+ sources_df = sources_df[
+ [
+ "id",
+ "uncertainty_ew",
+ "weight_ew",
+ "uncertainty_ns",
+ "weight_ns",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "forced",
+ "compactness",
+ "has_siblings",
+ "snr",
+ "image",
+ "datetime",
+ "source",
+ "ra",
+ "dec",
+ "ra_source",
+ "dec_source",
+ "d2d",
+ "dr",
+ "related",
+ "epoch",
+ "uncertainty_ew_source",
+ "uncertainty_ns_source",
+ ]
+ ]
# Create the unique skyc1_srcs dataframe.
skyc1_srcs = (
- sources_df[~sources_df['forced']]
- .sort_values(by='id')
- .drop('related', axis=1)
- .drop_duplicates('source')
+ sources_df[~sources_df["forced"]]
+ .sort_values(by="id")
+ .drop("related", axis=1)
+ .drop_duplicates("source")
).copy(deep=True)
# Get relations into the skyc1_srcs (as we only keep the first instance
# which does not have the relation information)
skyc1_srcs = skyc1_srcs.merge(
- prev_relations, how='left', left_on='source', right_index=True
+ prev_relations, how="left", left_on="source", right_index=True
)
# Need to break the pointer relationship between the related sources (
# deep=True copy does not truly copy mutable type objects)
relation_mask = skyc1_srcs.related.notna()
- relation_vals = skyc1_srcs.loc[relation_mask, 'related'].to_list()
- new_relation_vals = [x.copy() for x in relation_vals]
- skyc1_srcs.loc[relation_mask, 'related'] = new_relation_vals
+ relation_vals = skyc1_srcs.loc[relation_mask, "related"].to_list()
+ new_relation_vals = np.array([x.copy() for x in relation_vals], dtype='object')
+ skyc1_srcs.loc[relation_mask, "related"] = new_relation_vals
# Reorder so we don't mess up the dask metas.
- skyc1_srcs = skyc1_srcs[[
- 'id', 'ra', 'uncertainty_ew', 'weight_ew', 'dec', 'uncertainty_ns',
- 'weight_ns', 'flux_int', 'flux_int_err', 'flux_int_isl_ratio',
- 'flux_peak', 'flux_peak_err', 'flux_peak_isl_ratio', 'forced',
- 'compactness', 'has_siblings', 'snr', 'image', 'datetime', 'source',
- 'ra_source', 'dec_source', 'd2d', 'dr', 'related', 'epoch'
- ]].reset_index(drop=True)
+ skyc1_srcs = skyc1_srcs[
+ [
+ "id",
+ "ra",
+ "uncertainty_ew",
+ "weight_ew",
+ "dec",
+ "uncertainty_ns",
+ "weight_ns",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "forced",
+ "compactness",
+ "has_siblings",
+ "snr",
+ "image",
+ "datetime",
+ "source",
+ "ra_source",
+ "dec_source",
+ "d2d",
+ "dr",
+ "related",
+ "epoch",
+ ]
+ ].reset_index(drop=True)
# Finally move the source ra and dec back to the sources_df ra and dec
# columns
- sources_df['ra'] = sources_df['ra_source']
- sources_df['dec'] = sources_df['dec_source']
- sources_df['uncertainty_ew'] = sources_df['uncertainty_ew_source']
- sources_df['uncertainty_ns'] = sources_df['uncertainty_ns_source']
+ sources_df["ra"] = sources_df["ra_source"]
+ sources_df["dec"] = sources_df["dec_source"]
+ sources_df["uncertainty_ew"] = sources_df["uncertainty_ew_source"]
+ sources_df["uncertainty_ns"] = sources_df["uncertainty_ns_source"]
# Drop not needed columns for the sources_df.
- sources_df = sources_df.drop([
- 'uncertainty_ew_source', 'uncertainty_ns_source'
- ], axis=1).reset_index(drop=True)
+ sources_df = sources_df.drop(
+ ["uncertainty_ew_source", "uncertainty_ns_source"], axis=1
+ ).reset_index(drop=True)
return sources_df, skyc1_srcs
+def _convert_uuid_col_to_str(series: pd.Series) -> pd.Series:
+ """Converts a UUID column to a string column.
+
+ Args:
+ series: A pandas series containing UUIDs.
+
+ Returns:
+ A pandas series containing strings.
+ """
+ return series.astype(str)
+
+
def write_parquets(
- images: List[Image],
- skyregions: List[SkyRegion],
- bands: List[Band],
- run_path: str
+ images: List[Image], skyregions: List[SkyRegion], bands: List[Band], run_path: str
) -> pd.DataFrame:
"""
This function saves images, skyregions and bands to parquet files.
@@ -1686,24 +1609,21 @@ def write_parquets(
"""
# write images parquet file under pipeline run folder
images_df = pd.DataFrame(map(lambda x: x.__dict__, images))
- images_df = images_df.drop('_state', axis=1)
- images_df.to_parquet(
- os.path.join(run_path, 'images.parquet'),
- index=False
- )
+ images_df = images_df.drop("_state", axis=1)
+ for col in ["id", "skyreg_id", "band_id"]:
+ images_df[col] = _convert_uuid_col_to_str(images_df[col])
+ images_df.to_parquet(os.path.join(run_path, "images.parquet"), index=False)
+
# write skyregions parquet file under pipeline run folder
skyregs_df = pd.DataFrame(map(lambda x: x.__dict__, skyregions))
- skyregs_df = skyregs_df.drop('_state', axis=1)
- skyregs_df.to_parquet(
- os.path.join(run_path, 'skyregions.parquet'),
- index=False
- )
+ skyregs_df = skyregs_df.drop("_state", axis=1)
+ skyregs_df["id"] = _convert_uuid_col_to_str(skyregs_df["id"])
+ skyregs_df.to_parquet(os.path.join(run_path, "skyregions.parquet"), index=False)
+
# write skyregions parquet file under pipeline run folder
bands_df = pd.DataFrame(map(lambda x: x.__dict__, bands))
- bands_df = bands_df.drop('_state', axis=1)
- bands_df.to_parquet(
- os.path.join(run_path, 'bands.parquet'),
- index=False
- )
+ bands_df = bands_df.drop("_state", axis=1)
+ bands_df["id"] = _convert_uuid_col_to_str(bands_df["id"])
+ bands_df.to_parquet(os.path.join(run_path, "bands.parquet"), index=False)
return skyregs_df
diff --git a/vast_pipeline/plots.py b/vast_pipeline/plots.py
index 9068a6390..eaeb23e38 100644
--- a/vast_pipeline/plots.py
+++ b/vast_pipeline/plots.py
@@ -22,7 +22,7 @@
RadioButtonGroup,
Scatter,
WheelZoomTool,
- ColorBar
+ ColorBar,
)
from bokeh.models.formatters import DatetimeTickFormatter
from bokeh.layouts import row, Row, gridplot, Spacer, column
@@ -36,6 +36,7 @@
from typing import Tuple
from vast_pipeline.models import Measurement, Source
+from vast_pipeline.pipeline.utils import _convert_uuid_col_to_str
def plot_lightcurve(
@@ -72,25 +73,24 @@ def plot_lightcurve(
)
.values(
"id",
- "pk",
"taustart_ts",
"flux",
"flux_err_upper",
"flux_err_lower",
"forced",
- "name"
+ "name",
)
.order_by("taustart_ts")
)
# lightcurve required cols: taustart_ts, flux, flux_err_upper, flux_err_lower, forced
lightcurve = pd.DataFrame(measurements_qs)
+ lightcurve["id"] = _convert_uuid_col_to_str(lightcurve["id"])
+
# remap method values to labels to make a better legend
- lightcurve["method"] = lightcurve.forced.map(
- {True: "Forced", False: "Selavy"}
- )
- lightcurve['cutout'] = lightcurve['id'].apply(
- lambda x: f'/cutout/{x}/normal/?img_type=png'
+ lightcurve["method"] = lightcurve.forced.map({True: "Forced", False: "Selavy"})
+ lightcurve["cutout"] = lightcurve["id"].apply(
+ lambda x: f"/cutout/{x}/normal/?img_type=png"
)
lc_source = ColumnDataSource(lightcurve)
@@ -135,7 +135,7 @@ def plot_lightcurve(
)
)
fig_lc.xaxis.axis_label = "Datetime"
- fig_lc.xaxis[0].formatter = DatetimeTickFormatter(days="%F", hours='%H:%M')
+ fig_lc.xaxis[0].formatter = DatetimeTickFormatter(days="%F", hours="%H:%M")
fig_lc.yaxis.axis_label = (
"Peak flux (mJy/beam)" if use_peak_flux else "Integrated flux (mJy)"
)
@@ -162,9 +162,22 @@ def plot_lightcurve(
hover_tool_lc_callback = None
measurement_pairs = source.get_measurement_pairs()
if len(measurement_pairs) > 0:
- candidate_measurement_pairs_df = pd.DataFrame(measurement_pairs).query(
- f"m_{metric_suffix}.abs() >= {m_abs_min} and vs_{metric_suffix}.abs() >= {vs_abs_min}"
- ).reset_index()
+ candidate_measurement_pairs_df = (
+ pd.DataFrame(measurement_pairs)
+ .query(
+ f"m_{metric_suffix}.abs() >= {m_abs_min} and vs_{metric_suffix}.abs() >= {vs_abs_min}"
+ )
+ .reset_index()
+ )
+ candidate_measurement_pairs_df["measurement_a_id"] = _convert_uuid_col_to_str(
+ candidate_measurement_pairs_df["measurement_a_id"]
+ )
+ candidate_measurement_pairs_df["measurement_b_id"] = _convert_uuid_col_to_str(
+ candidate_measurement_pairs_df["measurement_b_id"]
+ )
+ candidate_measurement_pairs_df["source_id"] = _convert_uuid_col_to_str(
+ candidate_measurement_pairs_df["source_id"]
+ )
g = nx.Graph()
for _row in candidate_measurement_pairs_df.itertuples(index=False):
g.add_edge(_row.measurement_a_id, _row.measurement_b_id)
@@ -303,7 +316,9 @@ def plot_lightcurve(
""",
- formatters={"@taustart_ts": "datetime", },
+ formatters={
+ "@taustart_ts": "datetime",
+ },
mode="mouse",
callback=hover_tool_lc_callback,
)
@@ -311,6 +326,7 @@ def plot_lightcurve(
plot_row = row(fig_lc, fig_graph, sizing_mode="stretch_width")
plot_row.css_classes.append("mx-auto")
+
return plot_row
@@ -335,21 +351,17 @@ def fit_eta_v(
"""
if use_peak_flux:
- eta_label = 'eta_peak'
- v_label = 'v_peak'
+ eta_label = "eta_peak"
+ v_label = "v_peak"
else:
- eta_label = 'eta_int'
- v_label = 'v_int'
+ eta_label = "eta_int"
+ v_label = "v_int"
eta_log = np.log10(df[eta_label])
v_log = np.log10(df[v_label])
- eta_log_clipped = sigma_clip(
- eta_log, masked=False, stdfunc=mad_std, sigma=3
- )
- v_log_clipped = sigma_clip(
- v_log, masked=False, stdfunc=mad_std, sigma=3
- )
+ eta_log_clipped = sigma_clip(eta_log, masked=False, stdfunc=mad_std, sigma=3)
+ v_log_clipped = sigma_clip(v_log, masked=False, stdfunc=mad_std, sigma=3)
eta_fit_mean, eta_fit_sigma = norm.fit(eta_log_clipped)
v_fit_mean, v_fit_sigma = norm.fit(v_log_clipped)
@@ -358,10 +370,7 @@ def fit_eta_v(
def plot_eta_v_bokeh(
- source: Source,
- eta_sigma: float,
- v_sigma: float,
- use_peak_flux: bool = True
+ source: Source, eta_sigma: float, v_sigma: float, use_peak_flux: bool = True
) -> gridplot:
"""
Adapted from code written by Andrew O'Brien.
@@ -381,20 +390,23 @@ def plot_eta_v_bokeh(
Bokeh grid object containing figure.
"""
- df = pd.DataFrame(source.values(
- "id", "name", "eta_peak", "eta_int", "v_peak", "v_int", "n_meas_sel"
- ))
+ df = pd.DataFrame(
+ source.values(
+ "id", "name", "eta_peak", "eta_int", "v_peak", "v_int", "n_meas_sel"
+ )
+ )
- (
- eta_fit_mean, eta_fit_sigma,
- v_fit_mean, v_fit_sigma
- ) = fit_eta_v(df, use_peak_flux=use_peak_flux)
+ df["id"] = _convert_uuid_col_to_str(df["id"])
+
+ (eta_fit_mean, eta_fit_sigma, v_fit_mean, v_fit_sigma) = fit_eta_v(
+ df, use_peak_flux=use_peak_flux
+ )
eta_cutoff_log10 = eta_fit_mean + eta_sigma * eta_fit_sigma
v_cutoff_log10 = v_fit_mean + v_sigma * v_fit_sigma
- eta_cutoff = 10 ** eta_cutoff_log10
- v_cutoff = 10 ** v_cutoff_log10
+ eta_cutoff = 10**eta_cutoff_log10
+ v_cutoff = 10**v_cutoff_log10
# generate fitted curve data for plotting
eta_x = np.linspace(
@@ -410,12 +422,12 @@ def plot_eta_v_bokeh(
v_y = norm.pdf(v_x, loc=v_fit_mean, scale=v_fit_sigma)
if use_peak_flux:
- x_label = 'eta_peak'
- y_label = 'v_peak'
- title = 'Peak Flux'
+ x_label = "eta_peak"
+ y_label = "v_peak"
+ title = "Peak Flux"
else:
- x_label = 'eta_int'
- y_label = 'v_int'
+ x_label = "eta_int"
+ y_label = "v_int"
title = "Int. Flux"
# PLOTTING NOTE!
@@ -444,11 +456,10 @@ def plot_eta_v_bokeh(
cb_title = "Number of Selavy Measurements"
if df.shape[0] > settings.ETA_V_DATASHADER_THRESHOLD:
-
- hv.extension('bokeh')
+ hv.extension("bokeh")
# create dfs for bokeh and datashader
- mask = ((df[x_label] >= eta_cutoff) & (df[y_label] >= v_cutoff))
+ mask = (df[x_label] >= eta_cutoff) & (df[y_label] >= v_cutoff)
bokeh_df = df.loc[mask]
ds_df = df.loc[~mask]
@@ -456,11 +467,10 @@ def plot_eta_v_bokeh(
# create datashader version first
points = spread(
datashade(
- hv.Points(ds_df[[f"{x_label}_log10", f"{y_label}_log10"]]),
- cmap="Blues"
+ hv.Points(ds_df[[f"{x_label}_log10", f"{y_label}_log10"]]), cmap="Blues"
),
px=1,
- shape='square'
+ shape="square",
).opts(height=PLOT_HEIGHT, width=PLOT_WIDTH)
fig = hv.render(points)
@@ -468,11 +478,11 @@ def plot_eta_v_bokeh(
fig.xaxis.axis_label = x_axis_label
fig.yaxis.axis_label = y_axis_label
fig.aspect_scale = 1
- fig.sizing_mode = 'stretch_width'
+ fig.sizing_mode = "stretch_width"
fig.output_backend = "webgl"
# update the y axis default range
if bokeh_df.shape[0] > 0:
- fig.y_range.end = bokeh_df[f'{y_label}_log10'].max() + 0.2
+ fig.y_range.end = bokeh_df[f"{y_label}_log10"].max() + 0.2
cb_title += " (interactive points only)"
else:
@@ -499,7 +509,7 @@ def plot_eta_v_bokeh(
fill_color=cmap,
line_color=cmap,
marker="circle",
- size=5
+ size=5,
)
bokeh_g1 = fig.add_glyph(source_or_glyph=source, glyph=bokeh_points)
@@ -510,19 +520,16 @@ def plot_eta_v_bokeh(
("source", "@name"),
("\u03B7", f"@{x_label}"),
("V", f"@{y_label}"),
- ("id", "@id")
+ ("id", "@id"),
],
- mode='mouse'
+ mode="mouse",
)
fig.add_tools(hover)
- color_bar = ColorBar(
- color_mapper=cmap['transform'],
- title=cb_title
- )
+ color_bar = ColorBar(color_mapper=cmap["transform"], title=cb_title)
- fig.add_layout(color_bar, 'below')
+ fig.add_layout(color_bar, "below")
# axis histograms
# filter out any forced-phot points for these
@@ -539,7 +546,9 @@ def plot_eta_v_bokeh(
output_backend="webgl",
)
x_hist_data, x_hist_edges = np.histogram(
- df[f"{x_label}_log10"], density=True, bins=50,
+ df[f"{x_label}_log10"],
+ density=True,
+ bins=50,
)
x_hist.quad(
top=x_hist_data,
@@ -569,7 +578,9 @@ def plot_eta_v_bokeh(
output_backend="webgl",
)
y_hist_data, y_hist_edges = np.histogram(
- (df[f"{y_label}_log10"]), density=True, bins=50,
+ (df[f"{y_label}_log10"]),
+ density=True,
+ bins=50,
)
y_hist.quad(
right=y_hist_data,
@@ -602,7 +613,7 @@ def plot_eta_v_bokeh(
step=0.1,
value=eta_sigma,
title="\u03B7 sigma value",
- sizing_mode='stretch_width'
+ sizing_mode="stretch_width",
)
v_slider = Slider(
start=0,
@@ -610,29 +621,23 @@ def plot_eta_v_bokeh(
step=0.1,
value=v_sigma,
title="V sigma value",
- sizing_mode='stretch_width'
+ sizing_mode="stretch_width",
)
- labels = ['Peak', 'Integrated']
+ labels = ["Peak", "Integrated"]
active = 0 if use_peak_flux else 1
flux_choice_radio = RadioButtonGroup(
- labels=labels,
- active=active,
- sizing_mode='stretch_width'
+ labels=labels, active=active, sizing_mode="stretch_width"
)
- button = Button(
- label="Apply",
- button_type="primary",
- sizing_mode='stretch_width'
- )
+ button = Button(label="Apply", button_type="primary", sizing_mode="stretch_width")
button.js_on_click(
CustomJS(
args=dict(
eta_slider=eta_slider,
v_slider=v_slider,
button=button,
- flux_choice_radio=flux_choice_radio
+ flux_choice_radio=flux_choice_radio,
),
code="""
button.label = "Loading..."
@@ -641,7 +646,7 @@ def plot_eta_v_bokeh(
const peak = ["peak", "int"];
var fluxType = peak[flux_choice_radio.active];
getEtaVPlot(e, v, fluxType);
- """
+ """,
)
)
@@ -658,7 +663,7 @@ def plot_eta_v_bokeh(
eta_slider,
v_slider,
button,
- sizing_mode='stretch_width'
+ sizing_mode="stretch_width",
)
plot_column.css_classes.append("mx-auto")
@@ -677,7 +682,7 @@ def plot_eta_v_bokeh(
update_card(id);
getLightcurvePlot(id, fluxType);
});
- """
+ """,
)
tap = TapTool(callback=callback, renderers=[bokeh_g1])
diff --git a/vast_pipeline/serializers.py b/vast_pipeline/serializers.py
index 0670fd221..aa0b9e608 100644
--- a/vast_pipeline/serializers.py
+++ b/vast_pipeline/serializers.py
@@ -11,26 +11,26 @@
class RunSerializer(serializers.ModelSerializer):
- id = serializers.IntegerField(read_only=True)
+ id = serializers.UUIDField(read_only=True)
path = serializers.SerializerMethodField()
n_sources = serializers.IntegerField(read_only=True)
n_images = serializers.IntegerField(read_only=True)
n_selavy_measurements = serializers.IntegerField(read_only=True)
n_forced_measurements = serializers.IntegerField(read_only=True)
epoch_based = serializers.BooleanField(read_only=True)
- status = serializers.CharField(source='get_status_display')
+ status = serializers.CharField(source="get_status_display")
class Meta:
model = Run
- fields = '__all__'
- datatables_always_serialize = ('id',)
+ fields = "__all__"
+ datatables_always_serialize = ("id",)
def get_path(self, run):
return os.path.relpath(run.path)
class ImageSerializer(serializers.ModelSerializer):
- id = serializers.IntegerField(read_only=True)
+ id = serializers.UUIDField(read_only=True)
frequency = serializers.SerializerMethodField(read_only=True)
def get_frequency(self, obj):
@@ -39,24 +39,24 @@ def get_frequency(self, obj):
class Meta:
model = Image
fields = [
- 'id',
- 'name',
- 'datetime',
- 'frequency',
- 'ra',
- 'dec',
- 'rms_median',
- 'rms_min',
- 'rms_max',
- 'beam_bmaj',
- 'beam_bmin',
- 'beam_bpa'
+ "id",
+ "name",
+ "datetime",
+ "frequency",
+ "ra",
+ "dec",
+ "rms_median",
+ "rms_min",
+ "rms_max",
+ "beam_bmaj",
+ "beam_bmin",
+ "beam_bpa",
]
- datatables_always_serialize = ('id',)
+ datatables_always_serialize = ("id",)
class MeasurementSerializer(serializers.ModelSerializer):
- id = serializers.IntegerField(read_only=True)
+ id = serializers.UUIDField(read_only=True)
frequency = serializers.SerializerMethodField(read_only=True)
def get_frequency(self, obj):
@@ -65,53 +65,57 @@ def get_frequency(self, obj):
class Meta:
model = Measurement
fields = [
- 'id',
- 'name',
- 'ra',
- 'ra_err',
- 'uncertainty_ew',
- 'dec',
- 'dec_err',
- 'uncertainty_ns',
- 'flux_peak',
- 'flux_peak_err',
- 'flux_peak_isl_ratio',
- 'flux_int',
- 'flux_int_err',
- 'flux_int_isl_ratio',
- 'compactness',
- 'snr',
- 'has_siblings',
- 'forced',
- 'island_id',
- 'frequency'
+ "id",
+ "name",
+ "ra",
+ "ra_err",
+ "uncertainty_ew",
+ "dec",
+ "dec_err",
+ "uncertainty_ns",
+ "flux_peak",
+ "flux_peak_err",
+ "flux_peak_isl_ratio",
+ "flux_int",
+ "flux_int_err",
+ "flux_int_isl_ratio",
+ "compactness",
+ "snr",
+ "has_siblings",
+ "forced",
+ "island_id",
+ "frequency",
]
- datatables_always_serialize = ('id',)
+ datatables_always_serialize = ("id",)
class UserSerializer(serializers.ModelSerializer):
class Meta:
model = User
- fields =['username']
+ fields = ["username"]
class RunNameSerializer(serializers.ModelSerializer):
+ id = serializers.UUIDField(read_only=True)
+
class Meta:
model = Run
- fields = ['id', 'name']
- datatables_always_serialize = ('id',)
+ fields = ["id", "name"]
+ datatables_always_serialize = ("id",)
class SourceNameSerializer(serializers.ModelSerializer):
+ id = serializers.UUIDField(read_only=True)
run = RunNameSerializer()
+
class Meta:
model = Source
- fields= ['id', 'name', 'run']
- datatables_always_serialize = ('id',)
+ fields = ["id", "name", "run"]
+ datatables_always_serialize = ("id",)
class SourceSerializer(serializers.ModelSerializer):
- id = serializers.IntegerField(read_only=True)
+ id = serializers.UUIDField(read_only=True)
run = RunNameSerializer()
wavg_ra = serializers.SerializerMethodField()
wavg_dec = serializers.SerializerMethodField()
@@ -119,7 +123,7 @@ class SourceSerializer(serializers.ModelSerializer):
class Meta:
model = Source
fields = "__all__"
- datatables_always_serialize = ('id',)
+ datatables_always_serialize = ("id",)
def get_wavg_ra(self, source):
return deg2hms(source.wavg_ra, hms_format=True)
@@ -129,17 +133,18 @@ def get_wavg_dec(self, source):
class SourceFavSerializer(serializers.ModelSerializer):
+ id = serializers.UUIDField(read_only=True)
user = UserSerializer(read_only=True)
source = SourceNameSerializer(read_only=True)
deletefield = serializers.SerializerMethodField()
class Meta:
model = SourceFav
- fields = '__all__'
- datatables_always_serialize = ('id', 'source', 'user')
+ fields = "__all__"
+ datatables_always_serialize = ("id", "source", "user")
def get_deletefield(self, obj):
- redirect = reverse('vast_pipeline:api_sources_favs-detail', args=[obj.id])
+ redirect = reverse("vast_pipeline:api_sources_favs-detail", args=[obj.id])
string = (
f''
''
@@ -160,7 +165,9 @@ class RawImageSelavyListSerializer(serializers.Serializer):
class SesameResultSerializer(serializers.Serializer):
object_name = serializers.CharField(required=True)
- service = serializers.ChoiceField(choices=["all", "simbad", "ned", "vizier"], required=True)
+ service = serializers.ChoiceField(
+ choices=["all", "simbad", "ned", "vizier"], required=True
+ )
coord = serializers.CharField(read_only=True)
def validate(self, data):
@@ -175,7 +182,9 @@ def validate(self, data):
class CoordinateValidatorSerializer(serializers.Serializer):
coord = serializers.CharField(required=True)
- frame = serializers.ChoiceField(choices=frame_transform_graph.get_names(), required=True)
+ frame = serializers.ChoiceField(
+ choices=frame_transform_graph.get_names(), required=True
+ )
def validate(self, data):
try:
@@ -186,8 +195,8 @@ def validate(self, data):
class ExternalSearchSerializer(serializers.Serializer):
- """Serializer for external database cone search results, i.e. SIMBAD and NED.
- """
+ """Serializer for external database cone search results, i.e. SIMBAD and NED."""
+
object_name = serializers.CharField()
database = serializers.CharField(
help_text="Result origin database, e.g. SIMBAD or NED."
diff --git a/vast_pipeline/tests/test_pipeline/data/skyc1_srcs_in.csv b/vast_pipeline/tests/test_pipeline/data/skyc1_srcs_in.csv
index 09f25456f..fdffaa3fe 100644
--- a/vast_pipeline/tests/test_pipeline/data/skyc1_srcs_in.csv
+++ b/vast_pipeline/tests/test_pipeline/data/skyc1_srcs_in.csv
@@ -1,3 +1,3 @@
id,ra,dec,uncertainty_ew,uncertainty_ns,source,d2d,dr,related,epoch
-1,149.983325,-29.977778,0,2,1,0,0,,1
-2,150.020532,-30.016111,1,3,2,0,0,,1
+a7dbdab0-2b61-11ec-8d14-4a0009155011,149.983325,-29.977778,0,2,a8dbdab0-2b61-11ec-8d14-4a0009155011,0,0,,1
+b9fc9111-1234-1234-1234-123456789012,150.020532,-30.016111,1,3,b0fc9111-1234-1234-1234-123456789012,0,0,,1
diff --git a/vast_pipeline/tests/test_pipeline/data/skyc2_srcs_in.csv b/vast_pipeline/tests/test_pipeline/data/skyc2_srcs_in.csv
index 4eacc7baa..65897dfa7 100644
--- a/vast_pipeline/tests/test_pipeline/data/skyc2_srcs_in.csv
+++ b/vast_pipeline/tests/test_pipeline/data/skyc2_srcs_in.csv
@@ -1,4 +1,4 @@
id,ra,dec,uncertainty_ew,uncertainty_ns,source,d2d,dr,related,epoch
-1,149.983324,-29.977779,0.1,1.1,-1,0,0,,2
-2,149.983318,-30.016111,0.2,1.2,-1,0,0,,2
-3,150.020523,-30.016108,0.3,1.3,-1,0,0,,2
+a7dbdab0-2b61-11ec-8d14-4a0009155022,149.983324,-29.977779,0.1,1.1,,0,0,,2
+b0fc9222-1234-1234-1234-123456789012,149.983318,-30.016111,0.2,1.2,,0,0,,2
+b0fc9333-1234-1234-1234-123456789012,150.020523,-30.016108,0.3,1.3,,0,0,,2
diff --git a/vast_pipeline/tests/test_pipeline/data/sources_df_all.csv b/vast_pipeline/tests/test_pipeline/data/sources_df_all.csv
index f6c8767af..02af3c8cd 100644
--- a/vast_pipeline/tests/test_pipeline/data/sources_df_all.csv
+++ b/vast_pipeline/tests/test_pipeline/data/sources_df_all.csv
@@ -1,6 +1,6 @@
id,ra,dec,source,d2d,related,epoch
-1,149.983325,-29.977778,1,0,,1
-2,150.020532,-30.016111,2,0,,1
-1,149.983324,-29.977779,3,0,,2
-2,149.983318,-30.016111,4,0,,2
-3,150.020523,-30.016108,5,0,,2
+a7dbdab0-2b61-11ec-8d14-4a0009155011,149.983325,-29.977778,a8dbdab0-2b61-11ec-8d14-4a0009155011,0,,1
+b9fc9111-1234-1234-1234-123456789012,150.020532,-30.016111,b0fc9111-1234-1234-1234-123456789012,0,,1
+a7dbdab0-2b61-11ec-8d14-4a0009155022,149.983324,-29.977779,3,0,,2
+b0fc9222-1234-1234-1234-123456789012,149.983318,-30.016111,4,0,,2
+b0fc9333-1234-1234-1234-123456789012,150.020523,-30.016108,5,0,,2
diff --git a/vast_pipeline/tests/test_pipeline/data/sources_df_ass_in.csv b/vast_pipeline/tests/test_pipeline/data/sources_df_ass_in.csv
index ea44fdf8e..de67ebd7a 100644
--- a/vast_pipeline/tests/test_pipeline/data/sources_df_ass_in.csv
+++ b/vast_pipeline/tests/test_pipeline/data/sources_df_ass_in.csv
@@ -1,3 +1,3 @@
id,ra,dec,source,d2d,related,epoch
-1,149.983325,-29.977778,1,0,,1
-2,150.020532,-30.016111,2,0,,1
+a7dbdab0-2b61-11ec-8d14-4a0009155011,149.983325,-29.977778,a8dbdab0-2b61-11ec-8d14-4a0009155011,0,,1
+b9fc9111-1234-1234-1234-123456789012,150.020532,-30.016111,b0fc9111-1234-1234-1234-123456789012,0,,1
diff --git a/vast_pipeline/tests/test_pipeline/test_association.py b/vast_pipeline/tests/test_pipeline/test_association.py
index f477e87c5..6d7592da3 100644
--- a/vast_pipeline/tests/test_pipeline/test_association.py
+++ b/vast_pipeline/tests/test_pipeline/test_association.py
@@ -1,5 +1,6 @@
import ast
import os
+import numpy as np
import pandas as pd
from pathlib import Path
@@ -15,7 +16,6 @@
many_to_one_advanced,
basic_association,
advanced_association,
- _correct_parallel_source_ids
)
@@ -107,6 +107,14 @@ def test_duplicated_skyc2_nonempty(self):
sources_df: must contain at least all source ids present in skyc2_src,
rows with same id as duplicates in skyc2_srcs will be duplicated and
assigned to the new id.
+
+ source related d2d
+ 0 1 NaN 3.1
+ 1 9a327f7d-2d7a-4c18-a169-7ddb60fd984e [2] 4.2
+ 2 2 [9a327f7d-2d7a-4c18-a169-7ddb60fd984e] 2.3
+ 3 3 [a035b62f-57de-42c0-ba18-963dfe57a38c, 87f5aef... 3.3
+ 4 87f5aef3-8c04-4d42-b619-fd4c68fa0316 [3] 5.6
+ 5 a035b62f-57de-42c0-ba18-963dfe57a38c [3] 4.0
'''
skyc2_srcs = pd.read_csv(
os.path.join(DATA_PATH, 'skyc2_srcs_dup.csv'),
@@ -119,8 +127,28 @@ def test_duplicated_skyc2_nonempty(self):
skyc2_srcs, sources_df = one_to_many_basic(skyc2_srcs, sources_df)
- self.assertTrue(skyc2_srcs.equals(self.skyc2_srcs_out))
- self.assertTrue(sources_df.equals(self.sources_df_out))
+ # Note the testing is more complex here as the source indexes are now
+ # randomly generated UUIDs, so we can't test the source_skyc1 column
+ # directly. Instead we test that the source column is correct and that
+ # the related column is correct.
+
+ assert skyc2_srcs['source'].loc[[0, 2, 3]].to_list() == [1, 2, 3]
+ for source_id in skyc2_srcs['source'].loc[[1, 4, 5]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
+ assert skyc2_srcs["related"].fillna(-1).to_list() == [
+ -1,
+ [2],
+ [skyc2_srcs['source'].loc[1]],
+ [skyc2_srcs['source'].loc[5], skyc2_srcs['source'].loc[4]],
+ [3],
+ [3],
+ ]
+
+ assert sources_df['source'].loc[[0, 1, 2, 3, 4]].to_list() == [1, 2, 2, 2, 3]
+ for source_id in sources_df['source'].loc[[5, 6, 7, 8, 9]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
class OneToManyAdvancedTest(SimpleTestCase):
@@ -213,8 +241,35 @@ def test_method_advanced(self):
method='advanced'
)
- self.assertTrue(temp_srcs.equals(self.temp_srcs_advanced_out))
- self.assertTrue(sources_df.equals(self.sources_df_out))
+ # Note the testing is more complex here as the source indexes are now
+ # randomly generated UUIDs, so we can't test the source_skyc1 column
+ # directly. Instead we test that the source column is correct and that
+ # the related column is correct.
+
+ assert temp_srcs['source_skyc1'].loc[[0, 2, 5, 6, 7]].to_list() == [
+ 1, 2, 3, 7, 8
+ ]
+ for source_id in temp_srcs['source_skyc1'].loc[[1, 3, 4]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
+ assert temp_srcs["related_skyc1"].fillna(-1).to_list() == [
+ -1,
+ [2],
+ [temp_srcs['source_skyc1'].loc[1]],
+ [3],
+ [3],
+ [temp_srcs['source_skyc1'].loc[3], temp_srcs['source_skyc1'].loc[4]],
+ -1,
+ -1
+ ]
+
+ assert sources_df['source'].loc[[0, 1, 2, 3, 4]].to_list() == [
+ 1, 2, 2, 2, 3
+ ]
+ for source_id in sources_df['source'].loc[[5, 6, 7, 8, 9]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
+
def test_method_deruiter(self):
'''
@@ -246,8 +301,34 @@ def test_method_deruiter(self):
method='deruiter'
)
- self.assertTrue(temp_srcs.equals(self.temp_srcs_deruiter_out))
- self.assertTrue(sources_df.equals(self.sources_df_out))
+ # Note the testing is more complex here as the source indexes are now
+ # randomly generated UUIDs, so we can't test the source_skyc1 column
+ # directly. Instead we test that the source column is correct and that
+ # the related column is correct.
+
+ assert temp_srcs['source_skyc1'].loc[[0, 1, 3, 6, 7]].to_list() == [
+ 1, 2, 3, 7, 8
+ ]
+ for source_id in temp_srcs['source_skyc1'].loc[[2, 4, 5]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
+ assert temp_srcs["related_skyc1"].fillna(-1).to_list() == [
+ -1,
+ [temp_srcs['source_skyc1'].loc[2]],
+ [2],
+ [temp_srcs['source_skyc1'].loc[4], temp_srcs['source_skyc1'].loc[5]],
+ [3],
+ [3],
+ -1,
+ -1
+ ]
+
+ assert sources_df['source'].loc[[0, 1, 2, 3, 4]].to_list() == [
+ 1, 2, 2, 2, 3
+ ]
+ for source_id in sources_df['source'].loc[[5, 6, 7, 8, 9]]:
+ assert isinstance(source_id, str)
+ assert len(source_id) == 36
class ManyToManyAdvancedTest(SimpleTestCase):
@@ -387,7 +468,7 @@ class TestHelpers(SimpleTestCase):
Class which has some helper functions for testing.
'''
- def check_col(self, df1, df2, columns=['ra', 'dec', 'source', 'epoch']):
+ def check_col(self, df1, df2, columns=['ra', 'dec', 'epoch']):
'''
Function which checks that certain columns of two DataFrames are
equal for BasicAssociationTest and AdvancedAssociationTest.
@@ -476,11 +557,12 @@ def test_no_new_skyc2_srcs(self):
limit = Angle(10, unit='arcsec')
sources_df, skyc1_srcs = basic_association(
- sources_df, skyc1_srcs, skyc1, skyc2_srcs, skyc2, limit
+ sources_df, skyc1_srcs, skyc1, skyc2_srcs, skyc2, limit,
)
self.check_col(sources_df, self.sources_df_no_new)
self.check_col(skyc1_srcs, self.skyc1_srcs_in)
+ assert sources_df["source"].nunique() == 2
def test_zero_limit(self):
'''
@@ -517,6 +599,7 @@ def test_zero_limit(self):
self.check_col(sources_df, self.sources_df_all)
self.check_col(skyc1_srcs, self.skyc1_srcs_all)
+ assert sources_df['source'].nunique() == 5
def test_basic_association(self):
'''
@@ -553,6 +636,7 @@ def test_basic_association(self):
self.check_col(sources_df, self.sources_df_basic_out)
self.check_col(skyc1_srcs, self.skyc1_srcs_out)
+ assert sources_df["source"].nunique() == 3
class AdvancedAssociationTest(TestHelpers):
@@ -626,6 +710,7 @@ def test_no_new_skyc2_srcs(self):
self.check_col(sources_df, self.sources_df_no_new)
self.check_col(skyc1_srcs, self.skyc1_srcs_in)
+ assert sources_df['source'].nunique() == 2
def test_zero_bw_max(self):
'''
@@ -664,6 +749,7 @@ def test_zero_bw_max(self):
self.check_col(sources_df, self.sources_df_all)
self.check_col(skyc1_srcs, self.skyc1_srcs_all)
+ assert sources_df['source'].nunique() == 5
def test_advanced(self):
'''
@@ -702,6 +788,7 @@ def test_advanced(self):
self.check_col(sources_df, self.sources_df_advanced_out)
self.check_col(skyc1_srcs, self.skyc1_srcs_out)
+ assert sources_df['source'].nunique() == 3
def test_deruiter(self):
'''
@@ -742,56 +829,4 @@ def test_deruiter(self):
self.check_col(sources_df, self.sources_df_advanced_out)
self.check_col(skyc1_srcs, self.skyc1_srcs_out)
-
-
-class CorrectParallelSourceIdsTest(SimpleTestCase):
- '''
- Tests for _correct_parallel_souce_ids in association.py
- '''
-
- @classmethod
- def setUpClass(self):
- '''
- Load in correct outputs so inplace operations are tested.
- '''
- super().setUpClass()
- self.sources_df_in = pd.read_csv(
- os.path.join(DATA_PATH, 'sources_df_in.csv'),
- header=0,
- converters={'related': parse_or_nan}
- )
- self.sources_df_out_2 = pd.read_csv(
- os.path.join(DATA_PATH, 'sources_df_out_2.csv'),
- header=0,
- converters={'related': parse_or_nan}
- )
-
- def test_zero(self):
- '''
- Test _correct_parallel_source_ids doesn't change the input df when
- correction=0.
- '''
- df = pd.read_csv(
- os.path.join(DATA_PATH, 'sources_df_in.csv'),
- header=0,
- converters={'related': parse_or_nan}
- )
-
- df = _correct_parallel_source_ids(df, 0)
-
- self.assertTrue(df.equals(self.sources_df_in))
-
- def test_correct_parllel_source_ids(self):
- '''
- Test _correct_parallel_source_ids increases the numbers in the source
- and relate column by the correction amount.
- '''
- df = pd.read_csv(
- os.path.join(DATA_PATH, 'sources_df_in.csv'),
- header=0,
- converters={'related': parse_or_nan}
- )
-
- df = _correct_parallel_source_ids(df, 2)
-
- self.assertTrue(df.equals(self.sources_df_out_2))
+ assert sources_df['source'].nunique() == 3
diff --git a/vast_pipeline/tests/test_regression/property_check.py b/vast_pipeline/tests/test_regression/property_check.py
index d720cc6ed..aa32d9405 100644
--- a/vast_pipeline/tests/test_regression/property_check.py
+++ b/vast_pipeline/tests/test_regression/property_check.py
@@ -139,8 +139,9 @@ def test_known_in_forced(testcase: TestCase, forced: dict, sources: pd.DataFrame
The expected force extraction files for PSR J2129-04.
'''
sources = sources.reset_index()
+ sources
id_match = known_source(sources)
- source_id = sources.loc[id_match, 'id']
+ source_id = sources.loc[id_match, 'source']
meas_id = associations[
associations['source_id'] == source_id
]['meas_id'].values
diff --git a/vast_pipeline/urls.py b/vast_pipeline/urls.py
index b6e168990..373eed0c6 100644
--- a/vast_pipeline/urls.py
+++ b/vast_pipeline/urls.py
@@ -10,55 +10,63 @@
from vast_pipeline.models import Source
-app_name = 'vast_pipeline'
+app_name = "vast_pipeline"
register_converter(converters.RightAscensionConverter, "ra")
register_converter(converters.DeclinationConverter, "dec")
register_converter(converters.AngleConverter, "angle")
router = DefaultRouter()
-router.register(r'piperuns', views.RunViewSet, 'api_pipe_runs')
-router.register(r'images', views.ImageViewSet, 'api_images')
-router.register(r'measurements', views.MeasurementViewSet, 'api_measurements')
-router.register(r'sources', views.SourceViewSet, 'api_sources')
-router.register(r'rawimages', views.RawImageListSet, 'api_rawimages')
-router.register(r'runcfg', views.RunConfigSet, 'api_runcfg')
-router.register(r'runlog', views.RunLogSet, 'api_runlog')
-router.register(r'sourcesfavs', views.SourceFavViewSet, 'api_sources_favs')
-router.register(r'utils', views.UtilitiesSet, 'api_utils')
-router.register(r'plots', views.SourcePlotsSet, 'api_source_plots')
+router.register(r"piperuns", views.RunViewSet, "api_pipe_runs")
+router.register(r"images", views.ImageViewSet, "api_images")
+router.register(r"measurements", views.MeasurementViewSet, "api_measurements")
+router.register(r"sources", views.SourceViewSet, "api_sources")
+router.register(r"rawimages", views.RawImageListSet, "api_rawimages")
+router.register(r"runcfg", views.RunConfigSet, "api_runcfg")
+router.register(r"runlog", views.RunLogSet, "api_runlog")
+router.register(r"sourcesfavs", views.SourceFavViewSet, "api_sources_favs")
+router.register(r"utils", views.UtilitiesSet, "api_utils")
+router.register(r"plots", views.SourcePlotsSet, "api_source_plots")
urlpatterns = [
- path('piperuns/', views.RunIndex, name='run_index'),
- path('piperuns//', views.RunDetail, name='run_detail'),
- path('images/', views.ImageIndex, name='image_index'),
- re_path(
- r'^images/(?P\d+)(?:/(?P[\w]+))?/$',
+ path("piperuns/", views.RunIndex, name="run_index"),
+ path("piperuns//", views.RunDetail, name="run_detail"),
+ path("images/", views.ImageIndex, name="image_index"),
+ path(
+ "images//",
views.ImageDetail,
- name='image_detail'
+ name="image_detail",
),
- path('measurements/', views.MeasurementIndex, name='measurement_index'),
- re_path(
- r'^measurements/(?P\d+)(?:/(?P[\w]+))?/$',
+ path("measurements/", views.MeasurementIndex, name="measurement_index"),
+ path(
+ "measurements//",
views.MeasurementDetail,
- name='measurement_detail'
+ name="measurement_detail",
+ ),
+ path("sources/query/", views.SourceQuery, name="source_query"),
+ path("sources/query/plot/", views.SourceEtaVPlot, name="source_etav_plot"),
+ path(
+ "sources/query/plot/update//",
+ views.SourceEtaVPlotUpdate,
+ name="source_etav_plot_update",
),
- path('sources/query/', views.SourceQuery, name='source_query'),
- path('sources/query/plot/', views.SourceEtaVPlot, name='source_etav_plot'),
- path('sources/query/plot/update//', views.SourceEtaVPlotUpdate, name='source_etav_plot_update'),
- path('sources//', views.SourceDetail, name='source_detail'),
- path('sources/favs/', views.UserSourceFavsList, name='source_favs'),
+ path("sources//", views.SourceDetail, name="source_detail"),
+ path("sources/favs/", views.UserSourceFavsList, name="source_favs"),
path(
"sources/tags/autocomplete/",
tagulous.views.autocomplete_login,
kwargs={"tag_model": Source.tags.tag_model},
name="source_tags_autocomplete",
),
- path('cutout//', views.ImageCutout.as_view(), name='cutout'),
- path('cutout///', views.ImageCutout.as_view(), name='cutout'),
+ path("cutout//", views.ImageCutout.as_view(), name="cutout"),
+ path(
+ "cutout///",
+ views.ImageCutout.as_view(),
+ name="cutout",
+ ),
path(
- 'measurements//,,/region/',
+ "measurements//,,/region/",
views.MeasurementQuery.as_view(),
- name="measurements_region"
+ name="measurements_region",
),
- path('api/', include(router.urls)),
+ path("api/", include(router.urls)),
]
diff --git a/vast_pipeline/utils/utils.py b/vast_pipeline/utils/utils.py
index 6ea4904fe..142e6fa1c 100644
--- a/vast_pipeline/utils/utils.py
+++ b/vast_pipeline/utils/utils.py
@@ -236,7 +236,7 @@ def equ2gal(ra: float, dec: float) -> Tuple[float, float]:
Galactic longitude in degrees.
Galactic latitude in degrees.
"""
- c = SkyCoord(np.float(ra), np.float(dec), unit=(u.deg, u.deg), frame="icrs")
+ c = SkyCoord(float(ra), float(dec), unit=(u.deg, u.deg), frame="icrs")
l = c.galactic.l.deg
b = c.galactic.b.deg
diff --git a/vast_pipeline/utils/view.py b/vast_pipeline/utils/view.py
index ec25a26ed..8502a577f 100644
--- a/vast_pipeline/utils/view.py
+++ b/vast_pipeline/utils/view.py
@@ -1,4 +1,5 @@
"""Functions and variables used in pipeline/views.py."""
+from uuid import UUID
from vast_pipeline.models import SkyRegion
from typing import List, Dict, Optional, Any
@@ -6,181 +7,179 @@
# Defines the float format and scaling for all
# parameters presented in DATATABLES via AJAX call
FLOAT_FIELDS = {
- 'ra': {
- 'precision': 4,
- 'scale': 1,
+ "ra": {
+ "precision": 4,
+ "scale": 1,
},
- 'ra_err': {
- 'precision': 4,
- 'scale': 3600.,
+ "ra_err": {
+ "precision": 4,
+ "scale": 3600.0,
},
- 'uncertainty_ew': {
- 'precision': 4,
- 'scale': 3600.,
+ "uncertainty_ew": {
+ "precision": 4,
+ "scale": 3600.0,
+ },
+ "dec": {
+ "precision": 4,
+ "scale": 1,
+ },
+ "dec_err": {
+ "precision": 4,
+ "scale": 3600,
+ },
+ "uncertainty_ns": {
+ "precision": 4,
+ "scale": 3600.0,
+ },
+ "flux_int": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "flux_peak": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "flux_int_err": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "flux_peak_err": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "v_int": {
+ "precision": 2,
+ "scale": 1,
+ },
+ "eta_int": {
+ "precision": 2,
+ "scale": 1,
+ },
+ "v_peak": {
+ "precision": 2,
+ "scale": 1,
+ },
+ "eta_peak": {
+ "precision": 2,
+ "scale": 1,
+ },
+ "avg_flux_int": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "avg_flux_peak": {
+ "precision": 3,
+ "scale": 1,
+ },
+ "max_flux_peak": {
+ "precision": 3,
+ "scale": 1,
},
- 'dec': {
- 'precision': 4,
- 'scale': 1,
+ "min_flux_peak": {
+ "precision": 3,
+ "scale": 1,
},
- 'dec_err': {
- 'precision': 4,
- 'scale': 3600,
+ "min_flux_int": {
+ "precision": 3,
+ "scale": 1,
},
- 'uncertainty_ns': {
- 'precision': 4,
- 'scale': 3600.,
+ "max_flux_int": {
+ "precision": 3,
+ "scale": 1,
},
- 'flux_int': {
- 'precision': 3,
- 'scale': 1,
+ "min_flux_int_isl_ratio": {
+ "precision": 2,
+ "scale": 1,
},
- 'flux_peak': {
- 'precision': 3,
- 'scale': 1,
+ "min_flux_peak_isl_ratio": {
+ "precision": 2,
+ "scale": 1,
},
- 'flux_int_err': {
- 'precision': 3,
- 'scale': 1,
+ "flux_peak_isl_ratio": {
+ "precision": 2,
+ "scale": 1,
},
- 'flux_peak_err': {
- 'precision': 3,
- 'scale': 1,
+ "flux_int_isl_ratio": {
+ "precision": 2,
+ "scale": 1,
},
- 'v_int': {
- 'precision': 2,
- 'scale': 1,
+ "rms_median": {
+ "precision": 3,
+ "scale": 1,
},
- 'eta_int': {
- 'precision': 2,
- 'scale': 1,
+ "rms_min": {
+ "precision": 3,
+ "scale": 1,
},
- 'v_peak': {
- 'precision': 2,
- 'scale': 1,
+ "rms_max": {
+ "precision": 3,
+ "scale": 1,
},
- 'eta_peak': {
- 'precision': 2,
- 'scale': 1,
+ "new_high_sigma": {"precision": 3, "scale": 1},
+ "compactness": {
+ "precision": 3,
+ "scale": 1,
},
- 'avg_flux_int': {
- 'precision': 3,
- 'scale': 1,
+ "avg_compactness": {
+ "precision": 2,
+ "scale": 1,
},
- 'avg_flux_peak': {
- 'precision': 3,
- 'scale': 1,
+ "n_neighbour_dist": {
+ "precision": 2,
+ "scale": 60.0,
},
- 'max_flux_peak': {
- 'precision': 3,
- 'scale': 1,
+ "snr": {
+ "precision": 2,
+ "scale": 1,
},
- 'min_flux_peak': {
- 'precision': 3,
- 'scale': 1,
+ "min_snr": {
+ "precision": 2,
+ "scale": 1,
},
- 'min_flux_int': {
- 'precision': 3,
- 'scale': 1,
+ "max_snr": {
+ "precision": 2,
+ "scale": 1,
},
- 'max_flux_int': {
- 'precision': 3,
- 'scale': 1,
+ "beam_bmaj": {
+ "precision": 2,
+ "scale": 3600.0,
},
- 'min_flux_int_isl_ratio': {
- 'precision': 2,
- 'scale': 1,
+ "beam_bmin": {
+ "precision": 2,
+ "scale": 3600.0,
},
- 'min_flux_peak_isl_ratio': {
- 'precision': 2,
- 'scale': 1,
+ "beam_bpa": {
+ "precision": 2,
+ "scale": 1,
},
- 'flux_peak_isl_ratio': {
- 'precision': 2,
- 'scale': 1,
+ "frequency": {
+ "precision": 2,
+ "scale": 1,
},
- 'flux_int_isl_ratio': {
- 'precision': 2,
- 'scale': 1,
+ "vs_abs_significant_max_int": {
+ "precision": 2,
+ "scale": 1,
},
- 'rms_median': {
- 'precision': 3,
- 'scale': 1,
+ "vs_abs_significant_max_peak": {
+ "precision": 2,
+ "scale": 1,
},
- 'rms_min': {
- 'precision': 3,
- 'scale': 1,
+ "m_abs_significant_max_int": {
+ "precision": 2,
+ "scale": 1,
},
- 'rms_max': {
- 'precision': 3,
- 'scale': 1,
- },
- 'new_high_sigma': {
- 'precision': 3,
- 'scale': 1
- },
- 'compactness': {
- 'precision': 3,
- 'scale': 1,
- },
- 'avg_compactness': {
- 'precision': 2,
- 'scale': 1,
- },
- 'n_neighbour_dist': {
- 'precision': 2,
- 'scale': 60.,
- },
- 'snr': {
- 'precision': 2,
- 'scale': 1,
- },
- 'min_snr': {
- 'precision': 2,
- 'scale': 1,
- },
- 'max_snr': {
- 'precision': 2,
- 'scale': 1,
- },
- 'beam_bmaj': {
- 'precision': 2,
- 'scale': 3600.,
- },
- 'beam_bmin': {
- 'precision': 2,
- 'scale': 3600.,
- },
- 'beam_bpa': {
- 'precision': 2,
- 'scale': 1,
- },
- 'frequency': {
- 'precision': 2,
- 'scale': 1,
- },
- 'vs_abs_significant_max_int': {
- 'precision': 2,
- 'scale': 1,
- },
- 'vs_abs_significant_max_peak': {
- 'precision': 2,
- 'scale': 1,
- },
- 'm_abs_significant_max_int': {
- 'precision': 2,
- 'scale': 1,
- },
- 'm_abs_significant_max_peak': {
- 'precision': 2,
- 'scale': 1,
+ "m_abs_significant_max_peak": {
+ "precision": 2,
+ "scale": 1,
},
}
def generate_colsfields(
- fields: List[str], url_prefix_dict: Dict[str, str],
- not_orderable_col: Optional[List[str]]=None,
- not_searchable_col: Optional[List[str]]=None,
+ fields: List[str],
+ url_prefix_dict: Dict[str, str],
+ not_orderable_col: Optional[List[str]] = None,
+ not_searchable_col: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
"""
Generate data to be included in context for datatables.
@@ -211,64 +210,54 @@ def generate_colsfields(
not_searchable_col = []
for col in fields:
field2append = {}
- if col == 'name':
+ if col == "name":
field2append = {
- 'data': col, 'render': {
- 'url': {
- 'prefix': url_prefix_dict[col],
- 'col': 'name'
- }
- }
+ "data": col,
+ "render": {"url": {"prefix": url_prefix_dict[col], "col": "name"}},
}
- elif '.name' in col:
+ elif ".name" in col:
# this is for nested fields to build a render with column name
# and id in url. The API results should look like:
# {... , main_col : {'name': value, 'id': value, ... }}
- main_col = col.rsplit('.', 1)[0]
- field2append = {
- 'data': col,
- 'render': {
- 'url': {
- 'prefix': url_prefix_dict[col],
- 'col': main_col,
- 'nested': True,
- }
- }
- }
- elif col == 'n_sibl':
+ main_col = col.rsplit(".", 1)[0]
field2append = {
- 'data': col, 'render': {
- 'contains_sibl': {
- 'col': col
+ "data": col,
+ "render": {
+ "url": {
+ "prefix": url_prefix_dict[col],
+ "col": main_col,
+ "nested": True,
}
- }
+ },
}
+ elif col == "n_sibl":
+ field2append = {"data": col, "render": {"contains_sibl": {"col": col}}}
elif col in FLOAT_FIELDS:
field2append = {
- 'data': col,
- 'render': {
- 'float': {
- 'col': col,
- 'precision': FLOAT_FIELDS[col]['precision'],
- 'scale': FLOAT_FIELDS[col]['scale'],
+ "data": col,
+ "render": {
+ "float": {
+ "col": col,
+ "precision": FLOAT_FIELDS[col]["precision"],
+ "scale": FLOAT_FIELDS[col]["scale"],
}
- }
+ },
}
else:
- field2append = {'data': col}
+ field2append = {"data": col}
if col in not_orderable_col:
- field2append['orderable'] = False
+ field2append["orderable"] = False
if col in not_searchable_col:
- field2append['searchable'] = False
+ field2append["searchable"] = False
colsfields.append(field2append)
return colsfields
-def get_skyregions_collection(run_id: Optional[int]=None) -> Dict[str, Any]:
+def get_skyregions_collection(run_id: Optional[UUID] = None) -> Dict[str, Any]:
"""
Produce Sky region geometry shapes JSON object for d3-celestial.
@@ -284,37 +273,33 @@ def get_skyregions_collection(run_id: Optional[int]=None) -> Dict[str, Any]:
features = []
- for skr in skyregions:
- ra_fix = 360. if skr.centre_ra > 180. else 0.
+ for i, skr in enumerate(skyregions):
+ ra_fix = 360.0 if skr.centre_ra > 180.0 else 0.0
ra = skr.centre_ra - ra_fix
dec = skr.centre_dec
- width_ra = skr.width_ra / 2.
- width_dec = skr.width_dec / 2.
+ width_ra = skr.width_ra / 2.0
+ width_dec = skr.width_dec / 2.0
id = skr.id
features.append(
{
"type": "Feature",
"id": f"SkyRegion{id}",
- "properties": {
- "n": f"{id:02d}",
- "loc": [ra, dec]
- },
+ "properties": {"n": f"{i:02d}", "loc": [ra, dec]},
"geometry": {
"type": "MultiLineString",
- "coordinates": [[
- [ra+width_ra, dec+width_dec],
- [ra+width_ra, dec-width_dec],
- [ra-width_ra, dec-width_dec],
- [ra-width_ra, dec+width_dec],
- [ra+width_ra, dec+width_dec]
- ]]
- }
+ "coordinates": [
+ [
+ [ra + width_ra, dec + width_dec],
+ [ra + width_ra, dec - width_dec],
+ [ra - width_ra, dec - width_dec],
+ [ra - width_ra, dec + width_dec],
+ [ra + width_ra, dec + width_dec],
+ ]
+ ],
+ },
}
)
- skyregions_collection = {
- "type": "FeatureCollection",
- "features" : features
- }
+ skyregions_collection = {"type": "FeatureCollection", "features": features}
return skyregions_collection
diff --git a/vast_pipeline/views.py b/vast_pipeline/views.py
index 6245ed7dd..ce3d3e937 100644
--- a/vast_pipeline/views.py
+++ b/vast_pipeline/views.py
@@ -11,6 +11,7 @@
from glob import glob
from itertools import tee
from pathlib import Path
+from uuid import UUID, uuid4
from astropy.io import fits
from astropy.coordinates import SkyCoord, Angle
@@ -26,7 +27,11 @@
from django.db import transaction
from django.db.models import F, Count, QuerySet
from django.http import (
- FileResponse, Http404, HttpResponseRedirect, JsonResponse, HttpResponse
+ FileResponse,
+ Http404,
+ HttpResponseRedirect,
+ JsonResponse,
+ HttpResponse,
)
from django.shortcuts import render, redirect, get_object_or_404
from django.template.loader import render_to_string
@@ -42,9 +47,7 @@
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.viewsets import ModelViewSet, ViewSet
-from rest_framework.authentication import (
- SessionAuthentication, BasicAuthentication
-)
+from rest_framework.authentication import SessionAuthentication, BasicAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework import serializers
from django.contrib.postgres.aggregates.general import ArrayAgg
@@ -52,13 +55,24 @@
from vast_pipeline.plots import plot_lightcurve, plot_eta_v_bokeh
from vast_pipeline.models import (
- Comment, CommentableModel, Image, Measurement, Run, Source, SourceFav,
+ Comment,
+ CommentableModel,
+ Image,
+ Measurement,
+ Run,
+ Source,
+ SourceFav,
)
from vast_pipeline.serializers import (
- ImageSerializer, MeasurementSerializer, RunSerializer,
- SourceSerializer, RawImageSelavyListSerializer,
- SourceFavSerializer, SesameResultSerializer, CoordinateValidatorSerializer,
- ExternalSearchSerializer
+ ImageSerializer,
+ MeasurementSerializer,
+ RunSerializer,
+ SourceSerializer,
+ RawImageSelavyListSerializer,
+ SourceFavSerializer,
+ SesameResultSerializer,
+ CoordinateValidatorSerializer,
+ ExternalSearchSerializer,
)
from vast_pipeline.utils import external_query
from vast_pipeline.utils.utils import deg2dms, deg2hms, parse_coord, equ2gal
@@ -99,7 +113,8 @@ def _process_comment_form_get_comments(
comment_target_type = ContentType.objects.get_for_model(instance)
comments = Comment.objects.filter(
- content_type__pk=comment_target_type.id, object_id=instance.id,
+ content_type__pk=comment_target_type.id,
+ object_id=instance.id,
).order_by("datetime")
return comment_form, comments
@@ -107,25 +122,23 @@ def _process_comment_form_get_comments(
def Login(request):
context = {
- 'particlejs_conf_f': os.path.join(
- settings.STATIC_URL, 'js', 'particlesjs-config.json'
+ "particlejs_conf_f": os.path.join(
+ settings.STATIC_URL, "js", "particlesjs-config.json"
)
}
- return render(request, 'login.html', context)
+ return render(request, "login.html", context)
@login_required
def Home(request):
- context = {
- 'static_url': settings.STATIC_URL
- }
- return render(request, 'index.html', context)
+ context = {"static_url": settings.STATIC_URL}
+ return render(request, "index.html", context)
# Runs table
@login_required
def RunIndex(request):
- if request.method == 'POST':
+ if request.method == "POST":
# this POST section is for initialise a pipeline run
form = PipelineRunForm(request.POST)
if form.is_valid():
@@ -137,10 +150,7 @@ def RunIndex(request):
# Get the lists of user-provided file paths. These aren't in PipelineRunForm
# but rather manually defined in the template. We should fix that.
# TODO move file fields from the template into PipelineRunForm
- f_list = [
- 'image_files', 'selavy_files', 'background_files',
- 'noise_files'
- ]
+ f_list = ["image_files", "selavy_files", "background_files", "noise_files"]
for files in f_list:
cfg_data[files] = request.POST.getlist(files)
@@ -154,62 +164,57 @@ def RunIndex(request):
messages.success(
request,
mark_safe(
- f'Pipeline run {p_run.name} '
- 'initilialised successfully!'
- )
+ f"Pipeline run {p_run.name} "
+ "initilialised successfully!"
+ ),
)
- return redirect('vast_pipeline:run_detail', id=p_run.id)
+ return redirect("vast_pipeline:run_detail", id=p_run.id)
except Exception as e:
- messages.error(
- request,
- f'Issue in pipeline run initilisation: {e}'
- )
- return redirect('vast_pipeline:run_index')
+ messages.error(request, f"Issue in pipeline run initilisation: {e}")
+ return redirect("vast_pipeline:run_index")
else:
- messages.error(
- request,
- f'Form not valid: {form.errors}'
- )
- return redirect('vast_pipeline:run_index')
+ messages.error(request, f"Form not valid: {form.errors}")
+ return redirect("vast_pipeline:run_index")
- fields = [
- 'name',
- 'time',
- 'path',
- 'n_images',
- 'n_sources',
- 'status'
- ]
+ fields = ["name", "time", "path", "n_images", "n_sources", "status"]
colsfields = generate_colsfields(
fields,
- {'name': reverse('vast_pipeline:run_detail', args=[1])[:-2]}
+ {
+ "name": reverse(
+ "vast_pipeline:run_detail",
+ args=["e1f6bf88-4b07-4f28-bf9b-3ccf9d2ca813"],
+ )[:-37]
+ },
)
return render(
request,
- 'generic_table.html',
+ "generic_table.html",
{
- 'text': {
- 'title': 'Pipeline Runs',
- 'description': 'List of pipeline runs below',
- 'breadcrumb': {'title': 'Pipeline Runs', 'url': request.path},
+ "text": {
+ "title": "Pipeline Runs",
+ "description": "List of pipeline runs below",
+ "breadcrumb": {"title": "Pipeline Runs", "url": request.path},
},
- 'datatable': {
- 'api': (
- reverse('vast_pipeline:api_pipe_runs-list') +
- '?format=datatables'
+ "datatable": {
+ "api": (
+ reverse("vast_pipeline:api_pipe_runs-list") + "?format=datatables"
),
- 'colsFields': colsfields,
- 'colsNames': [
- 'Name', 'Run Datetime', 'Path', 'Nr Images',
- 'Nr Sources', 'Run Status'
+ "colsFields": colsfields,
+ "colsNames": [
+ "Name",
+ "Run Datetime",
+ "Path",
+ "Nr Images",
+ "Nr Sources",
+ "Run Status",
],
- 'search': True,
+ "search": True,
},
- 'runconfig' : settings.PIPE_RUN_CONFIG_DEFAULTS,
- 'max_piperun_images': settings.MAX_PIPERUN_IMAGES
- }
+ "runconfig": settings.PIPE_RUN_CONFIG_DEFAULTS,
+ "max_piperun_images": settings.MAX_PIPERUN_IMAGES,
+ },
)
@@ -219,9 +224,9 @@ class RunViewSet(ModelViewSet):
queryset = Run.objects.all()
serializer_class = RunSerializer
- @rest_framework.decorators.action(detail=True, methods=['get'])
+ @rest_framework.decorators.action(detail=True, methods=["get"])
def images(self, request, pk=None):
- qs = Image.objects.filter(run__id=pk).order_by('id')
+ qs = Image.objects.filter(run__id=pk).order_by("id")
qs = self.filter_queryset(qs)
page = self.paginate_queryset(qs)
if page is not None:
@@ -231,9 +236,9 @@ def images(self, request, pk=None):
serializer = ImageSerializer(qs, many=True)
return Response(serializer.data)
- @rest_framework.decorators.action(detail=True, methods=['get'])
+ @rest_framework.decorators.action(detail=True, methods=["get"])
def measurements(self, request, pk=None):
- qs = Measurement.objects.filter(image__run__in=[pk]).order_by('id')
+ qs = Measurement.objects.filter(image__run__in=[pk]).order_by("id")
qs = self.filter_queryset(qs)
page = self.paginate_queryset(qs)
if page is not None:
@@ -243,10 +248,8 @@ def measurements(self, request, pk=None):
serializer = MeasurementSerializer(qs, many=True)
return Response(serializer.data)
- @rest_framework.decorators.action(detail=True, methods=['post'])
- def run(
- self, request: Request, pk: Optional[int] = None
- ) -> HttpResponseRedirect:
+ @rest_framework.decorators.action(detail=True, methods=["post"])
+ def run(self, request: Request, pk: Optional[int] = None) -> HttpResponseRedirect:
"""
Launches a pipeline run using a Django Q cluster. Includes a check
on ownership or admin stataus of the user to make sure processing
@@ -265,84 +268,81 @@ def run(
"""
if not pk:
messages.error(
- request,
- 'Error in config write: Run pk parameter null or not passed'
+ request, "Error in config write: Run pk parameter null or not passed"
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
p_run = get_object_or_404(self.queryset, pk=pk)
except Exception as e:
- messages.error(request, f'Error in config write: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in config write: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# make sure that only the run creator or an admin can request the run
# to be processed.
if p_run.user != request.user and not request.user.is_staff:
- msg = 'You do not have permission to process this pipeline run!'
+ msg = "You do not have permission to process this pipeline run!"
messages.error(request, msg)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# check that it's not already running or queued
if p_run.status in ["RUN", "QUE", "RES"]:
msg = (
- f'{p_run.name} is already running, queued or restoring.'
- ' Please wait for the run to complete before trying to'
- ' submit again.'
- )
- messages.error(
- request,
- msg
+ f"{p_run.name} is already running, queued or restoring."
+ " Please wait for the run to complete before trying to"
+ " submit again."
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
if Run.objects.check_max_runs(settings.MAX_PIPELINE_RUNS):
msg = (
- 'The maximum number of simultaneous pipeline runs has been'
- f' reached ({settings.MAX_PIPELINE_RUNS})! Please try again'
- ' when other jobs have finished.'
+ "The maximum number of simultaneous pipeline runs has been"
+ f" reached ({settings.MAX_PIPELINE_RUNS})! Please try again"
+ " when other jobs have finished."
)
- messages.error(
- request,
- msg
- )
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
prev_status = p_run.status
try:
with transaction.atomic():
- p_run.status = 'QUE'
+ p_run.status = "QUE"
p_run.save()
- debug_flag = True if request.POST.get('debug', None) else False
- full_rerun = True if request.POST.get('fullReRun', None) else False
+ debug_flag = True if request.POST.get("debug", None) else False
+ full_rerun = True if request.POST.get("fullReRun", None) else False
async_task(
- 'vast_pipeline.management.commands.runpipeline.run_pipe',
- p_run.name, p_run.path, p_run, False, debug_flag,
- task_name=p_run.name, ack_failure=True, user=request.user,
- full_rerun=full_rerun, prev_ui_status=prev_status
+ "vast_pipeline.management.commands.runpipeline.run_pipe",
+ p_run.name,
+ p_run.path,
+ p_run,
+ False,
+ debug_flag,
+ task_name=p_run.name,
+ ack_failure=True,
+ user=request.user,
+ full_rerun=full_rerun,
+ prev_ui_status=prev_status,
)
msg = mark_safe(
- f'{p_run.name} successfully sent to the queue!
Refresh the'
- ' page to check the status.'
- )
- messages.success(
- request,
- msg
+ f"{p_run.name} successfully sent to the queue!
Refresh the"
+ " page to check the status."
)
+ messages.success(request, msg)
except Exception as e:
with transaction.atomic():
- p_run.status = 'ERR'
+ p_run.status = "ERR"
p_run.save()
- messages.error(request, f'Error in running pipeline: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in running pipeline: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
return HttpResponseRedirect(
- reverse('vast_pipeline:run_detail', args=[p_run.id])
+ reverse("vast_pipeline:run_detail", args=[p_run.id])
)
- @rest_framework.decorators.action(detail=True, methods=['post'])
+ @rest_framework.decorators.action(detail=True, methods=["post"])
def restore(
self, request: Request, pk: Optional[int] = None
) -> HttpResponseRedirect:
@@ -364,80 +364,70 @@ def restore(
"""
if not pk:
messages.error(
- request,
- 'Error in config write: Run pk parameter null or not passed'
+ request, "Error in config write: Run pk parameter null or not passed"
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
p_run = get_object_or_404(self.queryset, pk=pk)
except Exception as e:
- messages.error(request, f'Error in run fetch: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in run fetch: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# make sure that only the run creator or an admin can request the run
# to be processed.
if p_run.user != request.user and not request.user.is_staff:
- msg = 'You do not have permission to process this pipeline run!'
+ msg = "You do not have permission to process this pipeline run!"
messages.error(request, msg)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# check that it's not already running or queued
if p_run.status in ["RUN", "QUE", "RES", "INI"]:
msg = (
- f'{p_run.name} is already running, queued, restoring or is '
- 'only initialised. It cannot be restored at this time.'
- )
- messages.error(
- request,
- msg
+ f"{p_run.name} is already running, queued, restoring or is "
+ "only initialised. It cannot be restored at this time."
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
if Run.objects.check_max_runs(settings.MAX_PIPELINE_RUNS):
msg = (
- 'The maximum number of simultaneous pipeline runs has been'
- f' reached ({settings.MAX_PIPELINE_RUNS})! Please try again'
- ' when other jobs have finished.'
- )
- messages.error(
- request,
- msg
+ "The maximum number of simultaneous pipeline runs has been"
+ f" reached ({settings.MAX_PIPELINE_RUNS})! Please try again"
+ " when other jobs have finished."
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
prev_status = p_run.status
try:
- debug_flag = 3 if request.POST.get('restoreDebug', None) else 1
+ debug_flag = 3 if request.POST.get("restoreDebug", None) else 1
async_task(
- 'django.core.management.call_command',
- 'restorepiperun',
+ "django.core.management.call_command",
+ "restorepiperun",
p_run.path,
no_confirm=True,
- verbosity=debug_flag
+ verbosity=debug_flag,
)
msg = mark_safe(
- f'Restore {p_run.name} successfully sent to the queue!
Refresh the'
- ' page to check the status.'
- )
- messages.success(
- request,
- msg
+ f"Restore {p_run.name} successfully sent to the queue!
Refresh the"
+ " page to check the status."
)
+ messages.success(request, msg)
except Exception as e:
with transaction.atomic():
- p_run.status = 'ERR'
+ p_run.status = "ERR"
p_run.save()
- messages.error(request, f'Error in restoring run: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in restoring run: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
return HttpResponseRedirect(
- reverse('vast_pipeline:run_detail', args=[p_run.id])
+ reverse("vast_pipeline:run_detail", args=[p_run.id])
)
- @rest_framework.decorators.action(detail=True, methods=['post'])
+ @rest_framework.decorators.action(detail=True, methods=["post"])
def delete(
self, request: Request, pk: Optional[int] = None
) -> HttpResponseRedirect:
@@ -458,67 +448,58 @@ def delete(
"""
if not pk:
messages.error(
- request,
- 'Error in config write: Run pk parameter null or not passed'
+ request, "Error in config write: Run pk parameter null or not passed"
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
p_run = get_object_or_404(self.queryset, pk=pk)
except Exception as e:
- messages.error(request, f'Error in run fetch: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in run fetch: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# make sure that only the run creator or an admin can request the run
# to be processed.
if p_run.user != request.user and not request.user.is_staff:
- msg = 'You do not have permission to process this pipeline run!'
+ msg = "You do not have permission to process this pipeline run!"
messages.error(request, msg)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# check that it's not already running or queued
if p_run.status in ["RUN", "QUE", "RES"]:
msg = (
- f'{p_run.name} is already running, queued or restoring. '
- 'It cannot be deleted at this time.'
+ f"{p_run.name} is already running, queued or restoring. "
+ "It cannot be deleted at this time."
)
- messages.error(
- request,
- msg
- )
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
async_task(
- 'django.core.management.call_command',
- 'clearpiperun',
+ "django.core.management.call_command",
+ "clearpiperun",
p_run.path,
remove_all=True,
)
msg = mark_safe(
- f'Delete {p_run.name} successfully requested!
'
- ' Refresh the Pipeline Runs page for the deletion to take effect.'
- )
- messages.success(
- request,
- msg
+ f"Delete {p_run.name} successfully requested!
"
+ " Refresh the Pipeline Runs page for the deletion to take effect."
)
+ messages.success(request, msg)
except Exception as e:
with transaction.atomic():
- p_run.status = 'ERR'
+ p_run.status = "ERR"
p_run.save()
- messages.error(request, f'Error in deleting run: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in deleting run: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
- return HttpResponseRedirect(
- reverse('vast_pipeline:run_index')
- )
+ return HttpResponseRedirect(reverse("vast_pipeline:run_index"))
- @rest_framework.decorators.action(detail=True, methods=['post'])
+ @rest_framework.decorators.action(detail=True, methods=["post"])
def genarrow(
self, request: Request, pk: Optional[int] = None
- ) ->HttpResponseRedirect:
+ ) -> HttpResponseRedirect:
"""
Launches the create arrow files process for a pipeline run using
a Django Q cluster. Includes a check on ownership or admin status of
@@ -537,160 +518,158 @@ def genarrow(
"""
if not pk:
messages.error(
- request,
- 'Error in config write: Run pk parameter null or not passed'
+ request, "Error in config write: Run pk parameter null or not passed"
)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
p_run = get_object_or_404(self.queryset, pk=pk)
except Exception as e:
- messages.error(request, f'Error in run fetch: {e}')
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, f"Error in run fetch: {e}")
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# make sure that only the run creator or an admin can request the run
# to be processed.
if p_run.user != request.user and not request.user.is_staff:
- msg = 'You do not have permission to process this pipeline run!'
+ msg = "You do not have permission to process this pipeline run!"
messages.error(request, msg)
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
# check that it's not already running or queued
if p_run.status != "END":
msg = (
- f'{p_run.name} has not completed successfully.'
- ' The arrow files can only be generated after the run is'
- ' successful.'
+ f"{p_run.name} has not completed successfully."
+ " The arrow files can only be generated after the run is"
+ " successful."
)
- messages.error(
- request,
- msg
- )
- return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
+ messages.error(request, msg)
+ return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
try:
- overwrite_flag = True if request.POST.get('arrowOverwrite', None) else False
+ overwrite_flag = True if request.POST.get("arrowOverwrite", None) else False
async_task(
- 'django.core.management.call_command',
- 'createmeasarrow',
+ "django.core.management.call_command",
+ "createmeasarrow",
p_run.path,
overwrite=overwrite_flag,
- verbosity=3
+ verbosity=3,
)
msg = mark_safe(
- f'Generate the arrow files for {p_run.name} successfully requested!
'
- ' Refresh the page and check the generate arrow log output for the status of the process.'
- )
- messages.success(
- request,
- msg
+ f"Generate the arrow files for {p_run.name} successfully requested!