Skip to content

Commit

Permalink
Add job definition and loop control files.
Browse files Browse the repository at this point in the history
  • Loading branch information
@klendathu2k
klendathu2k committed Oct 16, 2024
1 parent f86e623 commit 91de565
Show file tree
Hide file tree
Showing 3 changed files with 352 additions and 0 deletions.
343 changes: 343 additions & 0 deletions run2pp/DST_STREAMING_EVENT_run2pp_ana441_2024p007.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,343 @@
#________________________________________________________________________________________________________DST_PHYSICS__
DST_STREAMING_EVENT_run2pp:

params:
name: DST_STREAMING_EVENT_run2pp
build: ana.441
build_name: ana441
dbtag: 2024p007
logbase : $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
outbase : $(name)_$(build)_$(tag)
script : run_cosmics.sh
payload : ./ProdFlow/run2pp/cosmics/
neventsper: 10000
comment : "---"
rsync : "./ProdFlow/run2pp/cosmics/*,cups.py,bachi.py,odbc.ini"
mem : 20000MB
# 20GB of memory is not a typo

input:
db: daqdb
direct_path: /sphenix/lustre01/sphnxpro/{mode}/*/physics/
query: |-
with run2pp as (
select 43249 as firstrun,
53880 as lastrun
),
fullrun as (
select
'daqdb/filelist' as source ,
runnumber ,
0 as segment ,
string_agg( distinct split_part(filename,'/',-1), ' ' ) as files ,
string_agg( distinct split_part(filename,'/',-1) || ':' || firstevent || ':' || lastevent, ' ' ) as fileranges
from
filelist,run2pp
where
(
(filename like '/bbox%/TPC%physics%.evt' and lastevent>2 ) or
(filename like '/bbox%/TPOT%physics%.evt' and lastevent>2 ) or
(filename like '/bbox%/physics_intt%.evt' and lastevent>2 ) or
(filename like '/bbox%/GL1_physics%.evt' and lastevent>2 ) or
(filename like '/bbox%/physics_mvtx%.evt' and lastevent>2 )
)
and runnumber>=run2pp.firstrun and runnumber<=run2pp.lastrun
{run_condition}
group by runnumber
having
every(transferred_to_sdcc) and
max(lastevent)>1000 and
sum( case when filename like '/bbox%/GL1_physics%' then 1 else 0 end )>0 and
(
sum( case when filename like '/bbox%/TPC%physics%' then 1 else 0 end )>0 or
sum( case when filename like '/bbox%/TPOT%physics%' then 1 else 0 end )>0 or
sum( case when filename like '/bbox%/physics%intt%' then 1 else 0 end )>0 or
sum( case when filename like '/bbox%/physics_mvtx%.evt' then 1 else 0 end )>0
)
order by runnumber
)
select *,'full run' as runtype from fullrun
;
# TODO: Need to add error checking to make sure that outdir, logdir, etc... are quoted properly. Else, this will cause problems with argument substitution
filesystem:
outdir : "/sphenix/lustre01/sphnxpro/physics/slurp/streaming/physics/$(build)_$(tag)/run_$(rungroup)"
logdir : "file:///sphenix/data/data02/sphnxpro/streaminglogs/$(build)_$(tag)/run_$(rungroup)"
histdir : "/sphenix/data/data02/sphnxpro/streamhist/$(build)_$(tag)/run_$(rungroup)"
condor : "/tmp/testlogs/$(build)_$(tag)/run_$(rungroup)"

#
# Again I note the need to ensure that the arguments are properly specified given the
# definition of the payload script.
#
job:
executable : "{payload}/run_cosmics.sh"
arguments : "$(nevents) {outbase} {logbase} $(run) $(seg) {outdir} $(build) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {comment} {histdir} {PWD} {rsync}"
output_destination : '{logdir}'
log : '{condor}/{logbase}.condor'
accounting_group : "group_sphenix.mdc2"
accounting_group_user : "sphnxpro"
priority : '4000'
request_xferslots: '0'



DST_TRKR_HIT_run2pp:
# DST_EVENT works from a pre-built set of run lists.
params:
name: DST_TRKR_HIT_run2pp
build: ana.441
build_name: ana441
dbtag: 2024p007
logbase : $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
outbase : $(name)_$(build)_$(tag)
script : run.sh
payload : ./ProdFlow/run2pp/TrackingProduction/
mem : 2048MB
rsync : "./ProdFlow/run2pp/TrackingProduction/*,cups.py,bachi.py,odbc.ini"
input : "DST_STREAMING_EVENT_run2pp_ana441_2024p007%"
mnrun : 51428
mxrun : 53880

input:
db: fc
query: |-
select
'filecatalog/datasets' as source ,
runnumber ,
segment ,
filename as files ,
filename || ':0:' || events as fileranges
from
datasets
where
filename like '{input}'
{run_condition}
and runnumber>={mnrun}
and runnumber<={mxrun}
order by runnumber
{limit_condition}
;
filesystem:
outdir : "/sphenix/lustre01/sphnxpro/physics/slurp/tracking/$(build)_$(tag)/run_$(rungroup)"
logdir : "file:///sphenix/data/data02/sphnxpro/trackinglogs/$(build)_$(tag)/run_$(rungroup)"
histdir : "/sphenix/data/data02/sphnxpro/hitsethist/$(build)_$(tag)/run_$(rungroup)"
condor : "/tmp/trkrogs/$(build)_$(tag)/run_$(rungroup)"

job:
executable : "{payload}/run.sh"
arguments : "$(nevents) {outbase} {logbase} $(run) $(seg) {outdir} $(build) $(tag) $(inputs) $(ranges) {logdir} {histdir} {PWD} {rsync}"
output_destination : '{logdir}'
log : '{condor}/{logbase}.condor'
accounting_group : "group_sphenix.mdc2"
accounting_group_user : "sphnxpro"
priority : '3800'






#_________________________________________________________________________________________________________________________________________
DST_TRKR_CLUSTER_run2pp:
# DST_EVENT works from a pre-built set of run lists.
params:
name: DST_TRKR_CLUSTER_run2pp
build: ana.441
build_name: ana441
dbtag: 2024p007
logbase : $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
outbase : $(name)_$(build)_$(tag)
script : run_job0.sh
payload : ./ProdFlow/run2pp/TrackingProduction/
mem : 2048MB
nevents : 0
rsync : "./ProdFlow/run2pp/TrackingProduction/*,cups.py,bachi.py,odbc.ini"
input : "DST_TRKR_HIT_run2pp_ana441_2024p007%"
mnrun : 51428
mxrun : 53880


input:
db: fc
query: |-
select
'filecatalog/datasets' as source ,
runnumber ,
segment ,
filename as files ,
filename || ':0:' || events as fileranges
from
datasets
where
filename like '{input}'
{run_condition}
and runnumber>={mnrun}
and runnumber<={mxrun}
order by runnumber
{limit_condition}
;
filesystem:
outdir : "/sphenix/lustre01/sphnxpro/physics/slurp/tracking/$(build)_$(tag)/run_$(rungroup)"
logdir : "file:///sphenix/data/data02/sphnxpro/trackinglogs/$(build)_$(tag)/run_$(rungroup)"
histdir : "/sphenix/data/data02/sphnxpro/clusterhist/$(build)_$(tag)/run_$(rungroup)"
condor : "/tmp/trkrlogs/$(build)_$(tag)/run_$(rungroup)"

job:
executable : "{payload}/run_job0.sh"
arguments : "{nevents} {outbase} {logbase} $(run) $(seg) {outdir} $(build) $(tag) $(inputs) $(ranges) {logdir} {histdir} {PWD} {rsync}"
output_destination : '{logdir}'
log : '{condor}/{logbase}.condor'
accounting_group : "group_sphenix.mdc2"
accounting_group_user : "sphnxpro"
priority : '3800'




#_________________________________________________________________________________________________________________________________________
DST_TRKR_SEED_run2pp:
# DST_EVENT works from a pre-built set of run lists.
params:
name: DST_TRKR_SEED_run2pp
build: ana.441
build_name: ana441
dbtag: 2024p007
logbase : $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
outbase : $(name)_$(build)_$(tag)
script : run_jobA.sh
payload : ./ProdFlow/run2pp/TrackingProduction/
mem : 2048MB
nevents : 0
rsync : "./ProdFlow/run2pp/TrackingProduction/*,cups.py,bachi.py,odbc.ini"
input : "DST_TRKR_CLUSTER_run2pp_ana441_2024p007%"
mnrun : 51428
mxrun : 53880


input:
db: fc
query: |-
select
'filecatalog/datasets' as source ,
runnumber ,
segment ,
filename as files ,
filename || ':0:' || events as fileranges
from
datasets
where
filename like '{input}'
{run_condition}
and runnumber>={mnrun}
and runnumber<={mxrun}
order by runnumber
{limit_condition}
;
filesystem:
outdir : "/sphenix/lustre01/sphnxpro/physics/slurp/tracking/$(build)_$(tag)/run_$(rungroup)"
logdir : "file:///sphenix/data/data02/sphnxpro/trackinglogs/$(build)_$(tag)/run_$(rungroup)"
histdir : "/sphenix/data/data02/sphnxpro/seedhist/$(build)_$(tag)/run_$(rungroup)"
condor : "/tmp/trkrlogs/$(build)_$(tag)/run_$(rungroup)"

job:
executable : "{payload}/run_jobA.sh"
arguments : "{nevents} {outbase} {logbase} $(run) $(seg) {outdir} $(build) $(tag) $(inputs) $(ranges) {logdir} {histdir} {PWD} {rsync}"
output_destination : '{logdir}'
log : '{condor}/{logbase}.condor'
accounting_group : "group_sphenix.mdc2"
accounting_group_user : "sphnxpro"
priority : '3800'


DST_TRKR_TRACKS_run2pp:
# DST_EVENT works from a pre-built set of run lists.
params:
name: DST_TRKR_TRACKS_run2pp
build: ana.441
build_name: ana441
dbtag: 2024p007
logbase : $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
outbase : $(name)_$(build)_$(tag)
script : run_jobC.sh
payload : ./ProdFlow/run2pp/TrackingProduction/
mem : 2048MB
nevents : 0
rsync : "./ProdFlow/run2pp/TrackingProduction/*,cups.py,bachi.py,odbc.ini"
seeds : "DST_TRKR_SEED_run2pp_ana441_2024p007%"
clusters: "DST_TRKR_CLUSTER_run2pp_ana441_2024p007%"
mnrun : 51428
mxrun : 53880




input:
db: fc
query: |-
select
'filecatalog/datasets' as source ,
runnumber ,
segment ,
string_agg( distinct split_part(filename,'/',-1), ' ' ) as files ,
string_agg( distinct split_part(filename,'/',-1) || ':' || 0 || ':' || -1, ' ' ) as fileranges
from
datasets
where
(
filename like '{seeds}' or filename like '{clusters}'
)
{run_condition}
and runnumber>={mnrun}
and runnumber<={mxrun}
group by runnumber,segment
having (
sum( case when filename like '{seeds}' then 1 else 0 end )>0 and
sum( case when filename like '{clusters}' then 1 else 0 end )>0
)
order by runnumber
{limit_condition}
;
;
filesystem:
outdir : "/sphenix/lustre01/sphnxpro/physics/slurp/tracking/$(build)_$(tag)/run_$(rungroup)"
logdir : "file:///sphenix/data/data02/sphnxpro/trackinglogs//$(build)_$(tag)/run_$(rungroup)"
histdir : "/sphenix/data/data02/sphnxpro/trackhist/$(build)_$(tag)/run_$(rungroup)"
condor : "/tmp/trkrlogs/$(build)_$(tag)/run_$(rungroup)"

job:
executable : "{payload}/run_jobC.sh"
arguments : "{nevents} {outbase} {logbase} $(run) $(seg) {outdir} $(build) $(tag) $(inputs) $(ranges) {logdir} {histdir} {PWD} {rsync}"
output_destination : '{logdir}'
log : '{condor}/{logbase}.condor'
accounting_group : "group_sphenix.mdc2"
accounting_group_user : "sphnxpro"
priority : '3800'








5 changes: 5 additions & 0 deletions run2pp/streaming_event_run2pp_ana441_2024p007.rules
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
DST_STREAMING_EVENT_run2pp
#DST_TRKR_HIT_run2pp
#DST_TRKR_CLUSTER_run2pp
#DST_TRKR_SEED_run2pp
#DST_TRKR_TRACKS_run2pp
4 changes: 4 additions & 0 deletions run2pp/tracking_run2pp_ana441_2024p007.rules
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
DST_TRKR_HIT_run2pp
DST_TRKR_CLUSTER_run2pp
DST_TRKR_SEED_run2pp
DST_TRKR_TRACKS_run2pp

0 comments on commit 91de565

Please sign in to comment.