diff --git a/stata_code/data_extraction_processing/extraction/get_ma_allocation.do b/stata_code/data_extraction_processing/extraction/get_ma_allocation.do index 57a0a3d..d644c4a 100644 --- a/stata_code/data_extraction_processing/extraction/get_ma_allocation.do +++ b/stata_code/data_extraction_processing/extraction/get_ma_allocation.do @@ -2,6 +2,18 @@ /* read in and store the MRIP location stuff. Put it in the file ma_site_allocation.dta */ clear #delimit; -odbc load, exec("select site_id, stock_region_calc from recdbs.mrip_ma_site_list;") $myNEFSC_USERS_conn; -#delimit cr -save "${data_raw}/ma_site_allocation.dta", replace \ No newline at end of file +odbc load, exec("select site_id, stock_region_calc from recdbs.mrip_ma_site_list;") $mysole_conn; + +/* will need to switch the connection string to $myNEFSC_USERS_conn soon */ + + +/* patch in the extra site_id */ + +qui count; +local newobs=`r(N)'+1; +set obs `newobs'; + +replace site_id=4434 if site_id==.; +replace stock_region_calc="NORTH" if site_id==4434; + +save "${data_raw}/ma_site_allocation.dta", replace; \ No newline at end of file diff --git a/stata_code/data_extraction_processing/extraction_wrapper.do b/stata_code/data_extraction_processing/extraction_wrapper.do index 897ede3..7929201 100644 --- a/stata_code/data_extraction_processing/extraction_wrapper.do +++ b/stata_code/data_extraction_processing/extraction_wrapper.do @@ -1,7 +1,7 @@ /* after running do $BLAST_MRIP */ /* get the mass location data from oracle */ -*do ${extraction_code}/get_ma_allocation.do +do ${extraction_code}/get_ma_allocation.do /* copying over partial years of data */ diff --git a/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_Open_Close_gom.do b/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_Open_Close_gom.do new file mode 100644 index 0000000..4573ecb --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_Open_Close_gom.do @@ -0,0 +1,243 @@ +/* This is a file that produces a dataset that contains #of fish encountered per trip. +This is a port of Scott's "cod_domain_length_freqs_by_wave_gom_2013.sas" + +This is a template program for estimating length frequencies +using the MRIP public-use datasets. + +The program is setup to use information in the trip_yyyyw +dataset to define custom domains. The length frequencies are +estimated within the domains by merging the trip information onto +the size_yyyyw datasets. + +Required input datasets: + trip_yyyyw + size_yyyyw + + +It looks like we also need to port cod_domain_length_freqs_b2_by_wave_gom_2013 as well + +There will be one output per variable and year in working directory: +"$my_common`myv'_b2_OpenClose_$working_year.dta" + + + +*/ + +/* General strategy +COMPUTE totals and std deviations for cod catch + + */ + clear + mata: mata clear + + + + +tempfile tl1 sl1 + +foreach file in $triplist{ + append using ${data_raw}/`file' +} +capture drop $drop_conditional + + +replace var_id=strat_id if strmatch(var_id,"") +sort year strat_id psu_id id_code +/* Deal with new variable names in the transition period + + */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } +} +save `tl1' +clear + + +foreach file in $b2list{ + append using ${data_raw}/`file' +} +cap drop $drop_conditional +/* Deal with new variable names in the transition period */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } + +} + + capture confirm variable wp_catch_chts + if _rc==0{ + drop wp_catch + rename wp_catch_chts wp_catch + else{ + } + +} + +sort year strat_id psu_id id_code +replace common=subinstr(lower(common)," ","",.) +save `sl1' + +use `tl1' +merge 1:m year strat_id psu_id id_code using `sl1', keep(1 3) +drop _merge +keep if year==$working_year +/* THIS IS THE END OF THE DATA MERGING CODE */ + + + /* ensure that domain is sub_reg=4 (New England), relevant states (MA, NH, ME), mode_fx =123, 457 */ + keep if sub_reg==4 + keep if st==23 | st==33 |st==25 + +/*This is the "full" mrip data */ +tempfile tc1 +save `tc1' + +/*classify as GOM or GB based on the ma_site_allocation.dta file */ +rename intsite site_id +sort site_id + + +merge m:1 site_id using "${data_raw}/ma_site_allocation.dta", keepusing(stock_region_calc) +rename site_id intsite +drop if _merge==2 +drop _merge + +/*classify into GOM or GBS */ +gen str3 area_s="OTH" + +replace area_s="GOM" if st==23 | st==33 +replace area_s="GOM" if st==25 & strmatch(stock_region_calc,"NORTH") +replace area_s="GBS" if st==25 & strmatch(stock_region_calc,"SOUTH") + + /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" */ + gen common_dom="Z" +if strmatch("$my_common","atlanticcod")==1{ + replace common_dom="C" if strmatch(sp_code,"8791030402") + } + + if strmatch("$my_common","haddock")==1{ + replace common_dom="H" if strmatch(sp_code,"8791031301") + } + + + + +tostring wave, gen(w2) +tostring year, gen(year2) + +destring month, gen(mymo) +drop month +tostring mymo, gen(month) +drop mymo + +tostring year, gen(myy) + + + + +/* l_in_bin already defined +gen l_in_bin=floor(lngth*0.03937) */ + +/* this might speed things up if I re-classify all length=0 for the species I don't care about */ +replace l_in_bin=0 if strmatch(common_dom, "Z")==1 + + +sort year w2 strat_id psu_id id_code + + +/* use unweighted for atlantic cod lengths */ +if "$my_common"=="atlanticcod"{ + svyset psu_id, strata(var_id) singleunit(certainty) + gen open=inlist(month,"9","10") + +} +if "$my_common"=="haddock"{ + svyset psu_id [pweight= wp_size], strata(var_id) singleunit(certainty) + gen open=!inlist(month,"3") + +} + +tostring open, gen(myo) + +gen my_dom_id_string=common_dom+"_"+area_s+"_"+myo + + + +local myv l_in_bin + + svy: tab `myv' my_dom_id_string, count + /*save some stuff + matrix of proportions, row names, column names, estimate of total population size*/ + mat eP=e(Prop) + mat eR=e(Row)' + mat eC=e(Col) + local PopN=e(N_pop) + + local mycolnames: colnames(eC) + mat colnames eP=`mycolnames' + + clear + /*read the eP into a dataset and convert proportion of population into numbers*/ + svmat eP, names(col) + foreach var of varlist *{ + replace `var'=`var'*`PopN' + } + /*read in the "row" */ + svmat eR + order eR + rename eR `myv' + + + + keep `myv' *GOM* + drop *Z_* + gen year=$working_year + + + qui desc + + foreach var of varlist *GOM*{ + tokenize `var', parse("_") + rename `var' `3'_`5' + } + reshape long GOM_, i(year l_in_bin) j(oo) + rename GOM_ count + gen str6 open="OPEN" if oo==1 + replace open="CLOSED" if oo==0 + + sort year open l_in_bin + order year open l_in_bin + drop oo + + + save "$my_outputdir/$my_common`myv'_b2_OpenClose_${working_year}.dta", replace + + + + + + + diff --git a/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_year_gom.do b/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_year_gom.do index aeb712d..e31b8e2 100644 --- a/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_year_gom.do +++ b/stata_code/data_extraction_processing/processing/annual/b2_length_freqs_by_year_gom.do @@ -167,6 +167,13 @@ replace l_in_bin=0 if strmatch(common_dom, "Z")==1 sort year w2 strat_id psu_id id_code svyset psu_id [pweight= wp_size], strata(var_id) singleunit(certainty) + +/* use unweighted for atlantic cod lengths */ +if "$my_common"=="atlanticcod"{ + svyset psu_id, strata(var_id) singleunit(certainty) +} + + local myv l_in_bin diff --git a/stata_code/data_extraction_processing/processing/annual/cod_haddock_directed_trips_annual.do b/stata_code/data_extraction_processing/processing/annual/cod_haddock_directed_trips_annual.do index 90dc469..583052b 100644 --- a/stata_code/data_extraction_processing/processing/annual/cod_haddock_directed_trips_annual.do +++ b/stata_code/data_extraction_processing/processing/annual/cod_haddock_directed_trips_annual.do @@ -201,10 +201,12 @@ svyset psu_id [pweight= wp_int], strata(var_id) singleunit(certainty) preserve sort my_dom_id year strat_id psu_id id_code local myvariables dtrip -local i=1 +local i=0 /* total with over() requires a numeric variable */ foreach var of local myvariables{ + local ++i + svy: total `var', over(my_dom_id) mat b`i'=e(b)' @@ -213,9 +215,7 @@ foreach var of local myvariables{ mat sub`i'=vecdiag(V)' mat colnames sub`i'=variance - local ++i } -local --i sort my_dom_id year duplicates drop my_dom_id, force keep my_dom_id year area_s month dom_id diff --git a/stata_code/data_extraction_processing/processing/annual/cod_weights_using_OpenClose.do b/stata_code/data_extraction_processing/processing/annual/cod_weights_using_OpenClose.do new file mode 100644 index 0000000..763bf13 --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/cod_weights_using_OpenClose.do @@ -0,0 +1,80 @@ + + +global cod_lwa 0.000005132 +global cod_lwb 3.1625 +global had_lwa 0.000009298 +global had_lwe 3.0205 +global lngcat_offset_cod 0.5 +global lngcat_offset_haddock 0.5 +global mt_to_kilo=1000 +global kilo_to_lbs=2.20462262 +global cm_to_inch=0.39370787 +global mortality_release=0.15 + + + + + +use "$annual/cod_size_class_OpenClose_$this_working_year.dta" if open=="OPEN", clear +expand 2 , gen(mark) +gen month=9 +replace month=10 if mark==1 + +drop mark +tempfile open +save `open' + + +use "$annual/cod_size_class_OpenClose_$this_working_year.dta" if open=="CLOSED", clear +expand 12 +bysort year open lngcat: gen month=_n +drop if inlist(month,9,10) + +append using `open' + +drop count +bysort year month open: egen tab1=total(ab1_count) +bysort year month open: egen tb2=total(b2_count) + +gen prob_ab1=ab1_count/tab1 +gen prob_b2=b2_count/tb2 + + +keep year month open lngcat prob* + +tempfile lengths +save `lengths', replace + + +use "${monthlydir}\atlanticcod_catch_$this_working_year.dta", clear +gen ab1=claim+harvest +rename release b2 + +gen str6 open="OPEN" if inlist(month,"09","10") +replace open="CLOSED" if open=="" +destring month, replace +keep year month ab1 b2 open +merge 1:m year month open using `lengths', keep(1 3) +gen ab1_count=ab1*prob_ab1 +gen b2_count=b2*prob_b2 +keep year month lngcat open ab1_count b2_count +sort year month open lngcat + + +gen weight_per_fish= $kilo_to_lbs*$cod_lwa*((lngcat+.5)/$cm_to_inch)^$cod_lwb +gen ab1mt=weight_per_fish*ab1_count/($mt_to_kilo*$kilo_to_lbs) +gen b2mt=weight_per_fish*b2_count/($mt_to_kilo*$kilo_to_lbs) + + + +collapse (sum) ab1_count b2_count ab1mt b2mt , by(year month) + +gen b2dead_mt=b2mt*$mortality_release + +format ab1_count b2_count %10.0fc + +format b2mt ab1mt b2dead_mt %6.2fc + +save "$annual\atlanticcod_weights_OpenClose_${this_working_year}.dta", replace + + diff --git a/stata_code/data_extraction_processing/processing/annual/domain_cod_Open_Close_catch_totals.do b/stata_code/data_extraction_processing/processing/annual/domain_cod_Open_Close_catch_totals.do new file mode 100644 index 0000000..099d3ab --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/domain_cod_Open_Close_catch_totals.do @@ -0,0 +1,182 @@ +/* This is a file that produces data on a, b1, b2, and other top-level catch statistics by wave + +This is a port of Scott's sas code + +*/ +version 12.1 + +/* General strategy +COMPUTE totals and std deviations for cod catch + + */ + mata: mata clear + +local my_common $species1 +tempfile tl1 cl1 + +clear +foreach file in $triplist{ + append using ${data_raw}/`file' +} +capture drop $drop_conditional + +sort year strat_id psu_id id_code +/* Deal with new variable names in the transition period + + */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } +} +save `tl1' + +clear + +foreach file in $catchlist{ + append using ${data_raw}/`file' +} +cap drop $drop_conditional +replace var_id=strat_id if strmatch(var_id,"") +replace wp_catch=wp_int if wp_catch==. +/* Deal with new variable names in the transition period */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } + +} + + capture confirm variable wp_catch_chts + if _rc==0{ + drop wp_catch + rename wp_catch_chts wp_catch + else{ + } + +} +sort year strat_id psu_id id_code +replace common=subinstr(lower(common)," ","",.) +save `cl1' +use `tl1' +merge 1:m year wave strat_id psu_id id_code using `cl1', keep(3) +drop _merge + +/* THIS IS THE END OF THE DATA MERGING CODE */ + + +/* ensure that domain is sub_reg=4 (New England), relevant states (MA, NH, ME), mode_fx =123, 457 */ +* keep if inlist(sub_reg,4,5) +* keep if st==23 | st==33 |st==25 + +/*This is the "full" mrip data */ +tempfile tc1 +save `tc1' + +/*classify as GOM or GB based on the ma_site_allocation.dta file */ +rename intsite site_id +sort site_id + + +merge m:1 site_id using "${data_raw}/ma_site_allocation.dta", keepusing(stock_region_calc) +rename site_id intsite +drop if _merge==2 +drop _merge + +/*classify into GOM or GBS */ +gen str3 area_s="OTH" + +replace area_s="GBS" if inlist(sub_reg,5 ,6 ,7) | inlist(st,9, 44) +replace area_s="GOM" if st==23 | st==33 +replace area_s="GOM" if st==25 & strmatch(stock_region_calc,"NORTH") +replace area_s="GBS" if st==25 & strmatch(stock_region_calc,"SOUTH") + + /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" */ + gen common_dom="zzzzzz" + replace common_dom=common if strmatch(common, "`my_common'") + + +tostring wave, gen(w2) +tostring year, gen(year2) +tostring year, gen(myy) + + + +gen open=inlist(month,"09","10") +tostring open, gen(myo) + +gen my_dom_id_string=year2+"_"+myo+"_"+area_s+"_"+myy+"_"+common_dom + +replace my_dom_id_string=ltrim(rtrim(my_dom_id_string)) +encode my_dom_id_string, gen(my_dom_id) +replace wp_catch=0 if wp_catch<=0 +sort year my_dom_id + + + +svyset psu_id [pweight= wp_catch], strata(var_id) singleunit(certainty) + + +local myvariables tot_cat claim harvest release +local i=0 +/* total with over() requires a numeric variable + + + + */ + +foreach var of local myvariables{ + local ++i + + svy: total `var', over(my_dom_id) + + mat b`i'=e(b)' + mat colnames b`i'=`var' + mat V=e(V) + +} +sort year my_dom_id +duplicates drop my_dom_id, force +keep my_dom_id year area_s month common_dom + +foreach j of numlist 1/`i'{ + svmat b`j', names(col) +} + +drop if strmatch(common_dom,"zzzzzz") +keep if strmatch(area_s,"GOM") +sort year area_s month common_dom +drop month +decode my_dom_id, gen(my_dom_id_string) +split my_dom_id_string, gen(stub) parse("_") + + + gen str6 open="OPEN" if stub2=="1" + replace open="CLOSED" if stub2=="0" + drop stub* + + +save "$my_outputdir/`my_common'_catch_OpenClose_$working_year.dta", replace + + + diff --git a/stata_code/data_extraction_processing/processing/annual/domain_cod_annual_catch_totals.do b/stata_code/data_extraction_processing/processing/annual/domain_cod_annual_catch_totals.do index 84b3fd1..59080a6 100644 --- a/stata_code/data_extraction_processing/processing/annual/domain_cod_annual_catch_totals.do +++ b/stata_code/data_extraction_processing/processing/annual/domain_cod_annual_catch_totals.do @@ -133,7 +133,7 @@ svyset psu_id [pweight= wp_catch], strata(var_id) singleunit(certainty) local myvariables tot_cat claim harvest release -local i=1 +local i=0 /* total with over() requires a numeric variable @@ -141,15 +141,15 @@ local i=1 */ foreach var of local myvariables{ + local ++i + svy: total `var', over(my_dom_id) mat b`i'=e(b)' mat colnames b`i'=`var' mat V=e(V) - local ++i } -local --i sort year my_dom_id duplicates drop my_dom_id, force keep my_dom_id year area_s month common_dom diff --git a/stata_code/data_extraction_processing/processing/annual/domain_haddock_Open_Close_catch_totals.do b/stata_code/data_extraction_processing/processing/annual/domain_haddock_Open_Close_catch_totals.do new file mode 100644 index 0000000..5f9f09d --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/domain_haddock_Open_Close_catch_totals.do @@ -0,0 +1,184 @@ +/* This is a file that produces data on a, b1, b2, and other top-level catch statistics by wave + +This is a port of Scott's sas code + +*/ +version 12.1 + +/* General strategy +COMPUTE totals and std deviations for cod catch + + */ + mata: mata clear + +local my_common $species2 + + +tempfile tl1 cl1 + +clear +foreach file in $triplist{ + append using ${data_raw}/`file' +} +capture drop $drop_conditional + +sort year strat_id psu_id id_code +/* Deal with new variable names in the transition period + + */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } +} +save `tl1' +clear + + +foreach file in $catchlist{ + append using ${data_raw}/`file' +} +cap drop $drop_conditional +replace var_id=strat_id if strmatch(var_id,"") +replace wp_catch=wp_int if wp_catch==. +/* Deal with new variable names in the transition period */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } + +} + + capture confirm variable wp_catch_chts + if _rc==0{ + drop wp_catch + rename wp_catch_chts wp_catch + else{ + } + +} +sort year strat_id psu_id id_code +replace common=subinstr(lower(common)," ","",.) +save `cl1' + +use `tl1' +merge 1:m year wave strat_id psu_id id_code using `cl1', keep(3) +drop _merge +/*ONLY keep trips for which there was catch>0 */ + + +/* THIS IS THE END OF THE DATA MERGING CODE */ + + +/* ensure that domain is sub_reg=4 (New England), relevant states (MA, NH, ME), mode_fx =123, 457 */ +* keep if inlist(sub_reg,4,5) +* keep if st==23 | st==33 |st==25 + +/*This is the "full" mrip data */ +tempfile tc1 +save `tc1' + +/*classify as GOM or GB based on the ma_site_allocation.dta file */ +rename intsite site_id +sort site_id + + +merge m:1 site_id using "${data_raw}/ma_site_allocation.dta", keepusing(stock_region_calc) +rename site_id intsite +drop if _merge==2 +drop _merge + +/*classify into GOM or GBS */ +gen str3 area_s="OTH" + +replace area_s="GBS" if inlist(sub_reg,5 ,6 ,7) | inlist(st,9, 44) +replace area_s="GOM" if st==23 | st==33 +replace area_s="GOM" if st==25 & strmatch(stock_region_calc,"NORTH") +replace area_s="GBS" if st==25 & strmatch(stock_region_calc,"SOUTH") + + /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" */ + gen common_dom="zzzzzz" + replace common_dom=common if strmatch(common, "`my_common'") + + +tostring wave, gen(w2) +tostring year, gen(year2) +tostring year, gen(myy) + + + +gen open=!inlist(month,"03") +tostring open, gen(myo) + +gen my_dom_id_string=year2+"_"+myo+"_"+area_s+"_"+myy+"_"+common_dom + +replace my_dom_id_string=ltrim(rtrim(my_dom_id_string)) +encode my_dom_id_string, gen(my_dom_id) +replace wp_catch=0 if wp_catch<=0 +sort year my_dom_id +svyset psu_id [pweight= wp_int], strata(strat_id) singleunit(certainty) +/*svyset psu_id [pweight= wp_int], strata(strat_id) singleunit(certainty) */ + + +svyset psu_id [pweight= wp_catch], strata(var_id) singleunit(certainty) + + +local myvariables tot_cat claim harvest release +local i=0 + + +/* total with over() requires a numeric variable + +*/ + +foreach var of local myvariables{ + local ++i + svy: total `var', over(my_dom_id) + + mat b`i'=e(b)' + mat colnames b`i'=`var' + mat V=e(V) + +} +sort year my_dom_id +duplicates drop my_dom_id, force +keep my_dom_id year area_s month common_dom + +foreach j of numlist 1/`i'{ + svmat b`j', names(col) +} + +drop if strmatch(common_dom,"zzzzzz") +keep if strmatch(area_s,"GOM") +sort year area_s month common_dom +drop month +decode my_dom_id, gen(my_dom_id_string) +split my_dom_id_string, gen(stub) parse("_") + + + gen str6 open="OPEN" if stub2=="1" + replace open="CLOSED" if stub2=="0" + drop stub* + + +save "$my_outputdir/`my_common'_catch_OpenClose_$working_year.dta", replace diff --git a/stata_code/data_extraction_processing/processing/annual/domain_haddock_annual_catch_totals.do b/stata_code/data_extraction_processing/processing/annual/domain_haddock_annual_catch_totals.do index 2d9fe13..fcfef85 100644 --- a/stata_code/data_extraction_processing/processing/annual/domain_haddock_annual_catch_totals.do +++ b/stata_code/data_extraction_processing/processing/annual/domain_haddock_annual_catch_totals.do @@ -15,7 +15,7 @@ local my_common $species2 tempfile tl1 cl1 - +clear foreach file in $triplist{ append using ${data_raw}/`file' } @@ -81,7 +81,7 @@ replace common=subinstr(lower(common)," ","",.) save `cl1' use `tl1' -merge 1:m year strat_id psu_id id_code using `cl1', keep(3) +merge 1:m year wave strat_id psu_id id_code using `cl1', keep(3) drop _merge /*ONLY keep trips for which there was catch>0 */ @@ -138,7 +138,7 @@ svyset psu_id [pweight= wp_catch], strata(var_id) singleunit(certainty) local myvariables tot_cat claim harvest release -local i=1 +local i=0 /* total with over() requires a numeric variable @@ -146,15 +146,14 @@ local i=1 */ foreach var of local myvariables{ + local ++i svy: total `var', over(my_dom_id) mat b`i'=e(b)' mat colnames b`i'=`var' mat V=e(V) - local ++i } -local --i sort year my_dom_id duplicates drop my_dom_id, force keep my_dom_id year area_s month common_dom diff --git a/stata_code/data_extraction_processing/processing/annual/length_freqs_by_Open_Close_gom.do b/stata_code/data_extraction_processing/processing/annual/length_freqs_by_Open_Close_gom.do new file mode 100644 index 0000000..2ccb2d9 --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/length_freqs_by_Open_Close_gom.do @@ -0,0 +1,250 @@ + /* This is a file that produces a dataset that contains #of fish encountered per trip. +This is a port of Scott's "cod_domain_length_freqs_by_wave_gom_2013.sas" + +This is a template program for estimating length frequencies +using the MRIP public-use datasets. + +The program is setup to use information in the trip_yyyyw +dataset to define custom domains. The length frequencies are +estimated within the domains by merging the trip information onto +the size_yyyyw datasets. + +Required input datasets: + trip_yyyyw + size_yyyyw + + +It looks like we also need to port cod_domain_length_freqs_b2_by_wave_gom_2013 as well + +There will be one output per variable and year in working directory: +"$my_common`myv'_a1b1$working_year.dta" + + + +*/ + +/* General strategy +COMPUTE totals and std deviations for cod catch + + */ + clear + + mata: mata clear + +tempfile tl1 sl1 + +foreach file in $triplist{ + append using ${data_raw}/`file' +} +capture drop $drop_conditional + +replace var_id=strat_id if strmatch(var_id,"") +sort year strat_id psu_id id_code +/* Deal with new variable names in the transition period + + */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } +} +save `tl1' +clear + + +foreach file in $sizelist{ + append using ${data_raw}/`file' +} +cap drop $drop_conditional +/* Deal with new variable names in the transition period */ + + capture confirm variable wp_int_chts + if _rc==0{ + drop wp_int + rename wp_int_chts wp_int + else{ + } +} + capture confirm variable wp_size_chts + if _rc==0{ + drop wp_size + rename wp_size_chts wp_size + else{ + } + +} + + capture confirm variable wp_catch_chts + if _rc==0{ + drop wp_catch + rename wp_catch_chts wp_catch + else{ + } + +} + +sort year strat_id psu_id id_code +replace var_id=strat_id if strmatch(var_id,"") + +replace common=subinstr(lower(common)," ","",.) +save `sl1' + +use `tl1' +merge 1:m year strat_id psu_id id_code using `sl1', keep(1 3) +drop _merge +keep if year==$working_year +/* THIS IS THE END OF THE DATA MERGING CODE */ + + + /* ensure that domain is sub_reg=4 (New England), relevant states (MA, NH, ME), mode_fx =123, 457 */ + keep if sub_reg==4 + keep if st==23 | st==33 |st==25 + +/*This is the "full" mrip data */ +tempfile tc1 +save `tc1' + +/*classify as GOM or GB based on the ma_site_allocation.dta file */ +rename intsite site_id +sort site_id + + +merge m:1 site_id using "${data_raw}/ma_site_allocation.dta", keepusing(stock_region_calc) +rename site_id intsite +drop if _merge==2 +drop _merge + +/*classify into GOM or GBS */ +gen str3 area_s="OTH" + +replace area_s="GOM" if st==23 | st==33 +replace area_s="GOM" if st==25 & strmatch(stock_region_calc,"NORTH") +replace area_s="GBS" if st==25 & strmatch(stock_region_calc,"SOUTH") + + /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" */ + gen common_dom="Z" +if strmatch("$my_common","atlanticcod")==1{ + replace common_dom="C" if strmatch(sp_code,"8791030402") + } + + if strmatch("$my_common","haddock")==1{ + replace common_dom="H" if strmatch(sp_code,"8791031301") + } + + + + +tostring wave, gen(w2) +tostring year, gen(year2) + +destring month, gen(mymo) +drop month +tostring mymo, gen(month) +drop mymo + +tostring year, gen(myy) + + + + +/* l_in_bin already defined +gen l_in_bin=floor(lngth*0.03937) */ + +/* this might speed things up if I re-classify all length=0 for the species I don't care about */ +replace l_in_bin=0 if strmatch(common_dom, "Z")==1 + + +sort year w2 strat_id psu_id id_code + + + + + + +pause + + + + +/* use unweighted for atlantic cod lengths */ +if "$my_common"=="atlanticcod"{ + svyset psu_id, strata(var_id) singleunit(certainty) + gen open=inlist(month,"9","10") + +} +if "$my_common"=="haddock"{ + svyset psu_id [pweight= wp_size], strata(var_id) singleunit(certainty) + gen open=!inlist(month,"3") + +} + +tostring open, gen(myo) + +gen my_dom_id_string=common_dom+"_"+area_s+"_"+myo + + + +local myv l_in_bin + + svy: tab `myv' my_dom_id_string, count + /*save some stuff + matrix of proportions, row names, column names, estimate of total population size*/ + mat eP=e(Prop) + mat eR=e(Row)' + mat eC=e(Col) + local PopN=e(N_pop) + + local mycolnames: colnames(eC) + mat colnames eP=`mycolnames' + + clear + /*read the eP into a dataset and convert proportion of population into numbers*/ + svmat eP, names(col) + foreach var of varlist *{ + replace `var'=`var'*`PopN' + } + /*read in the "row" */ + svmat eR + order eR + rename eR `myv' + + + + keep `myv' *GOM* + drop *Z_* + gen year=$working_year + + + qui desc + + foreach var of varlist *GOM*{ + tokenize `var', parse("_") + rename `var' `3'_`5' + } + reshape long GOM_, i(year l_in_bin) j(oo) + rename GOM_ count + gen str6 open="OPEN" if oo==1 + replace open="CLOSED" if oo==0 + + sort year open l_in_bin + order year open l_in_bin + drop oo + + + save "$my_outputdir/$my_common`myv'_a1b1_OpenClose_${working_year}.dta", replace + +clear + + + + diff --git a/stata_code/data_extraction_processing/processing/annual/length_freqs_by_year_gom.do b/stata_code/data_extraction_processing/processing/annual/length_freqs_by_year_gom.do index e0609bc..bbc789b 100644 --- a/stata_code/data_extraction_processing/processing/annual/length_freqs_by_year_gom.do +++ b/stata_code/data_extraction_processing/processing/annual/length_freqs_by_year_gom.do @@ -131,16 +131,17 @@ replace area_s="GOM" if st==23 | st==33 replace area_s="GOM" if st==25 & strmatch(stock_region_calc,"NORTH") replace area_s="GBS" if st==25 & strmatch(stock_region_calc,"SOUTH") - /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" use the id_code*/ + /* classify catch into the things I care about (common==$mycommon) and things I don't care about "ZZZZZZZZ" */ gen common_dom="Z" if strmatch("$my_common","atlanticcod")==1{ - replace common_dom="COD" if strmatch(sp_code,"8791030402") + replace common_dom="C" if strmatch(sp_code,"8791030402") } if strmatch("$my_common","haddock")==1{ - replace common_dom="HAD" if strmatch(sp_code,"8791031301") + replace common_dom="H" if strmatch(sp_code,"8791031301") } + tostring wave, gen(w2) tostring year, gen(year2) @@ -177,6 +178,12 @@ replace l_in_bin=0 if strmatch(common_dom, "Z")==1 sort year2 area_s w2 strat_id psu_id id_code common_dom svyset psu_id [pweight= wp_size], strata(var_id) singleunit(certainty) +/* use unweighted for atlantic cod lengths */ +if "$my_common"=="atlanticcod"{ + svyset psu_id, strata(var_id) singleunit(certainty) +} + + local myv l_in_bin diff --git a/stata_code/data_extraction_processing/processing/annual/process_b2_cod_OpenClose.do b/stata_code/data_extraction_processing/processing/annual/process_b2_cod_OpenClose.do new file mode 100644 index 0000000..0040ccd --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/process_b2_cod_OpenClose.do @@ -0,0 +1,93 @@ + + +/* in order to get the right distribution of lengths, you have to multiply the B2 length proportions by the number of B2's in each year. You have to multiply the A+B1 length proportions by the number of A+B1's in a year. + +Then you have to add these up. +*/ + + +/* process the b2 cod */ +use "$my_outputdir/atlanticcod_landings_OpenClose_$working_year.dta", clear +gen ab1=landings +keep year open ab1 +tempfile claim_harvest +sort year +save `claim_harvest' + +use "$my_outputdir/atlanticcod_landings_OpenClose_$working_year.dta", clear +rename b2 release +keep year open release +tempfile cm +sort year +save `cm' + + + +use "$my_outputdir/cod_b2_OpenClose_$working_year.dta", clear + +sort year +merge m:1 year open using `cm' +replace release=0 if release==. +drop _merge + + +tempvar tt +bysort year open : egen `tt'=total(count) +gen prob=count/`tt' + +gen b2_count=prob*release + +keep year open l_in_bin b2_count + +sort year open l_in_bin +/* save */ +save "$my_outputdir/atlanticcod_b2_counts_OpenClose_$working_year.dta", replace + + + + +use "$my_outputdir/cod_ab1_OpenClose_$working_year.dta", clear + + +sort year + +cap drop _merge +merge m:1 year open using `claim_harvest' + + +tempvar ttt +bysort year open : egen `ttt'=total(count) +gen prob=count/`ttt' + +gen ab1_count=prob*ab1 + +keep year open l_in_bin ab1_count + +sort year open l_in_bin + +merge 1:1 year open l_in_bin using "$my_outputdir/atlanticcod_b2_counts_OpenClose_$working_year.dta" +replace ab1_count=0 if ab1_count==. +replace b2_count=0 if b2_count==. +gen countnumbersoffish=round(ab1_count+b2_count) +tempfile tth1 +save `tth1' + +keep year open l_in_bin countnumbersoffish ab1_count b2_count + +rename l_in_bin lngcatinches +sort year open lngcatinches + +label var lngcatinches "length in inches" + + + +save "$my_outputdir/cod_size_class_OpenClose_$working_year.dta", replace + +/* compute the numbers of sub-legal fish retained +use `tt1', clear +keep year l_in_bin ab1_count countnumbers + +save "$my_outputdir/cod_sublegals_annual_$working_year.dta", replace */ + + + diff --git a/stata_code/data_extraction_processing/processing/annual/process_b2_haddock_OpenClose.do b/stata_code/data_extraction_processing/processing/annual/process_b2_haddock_OpenClose.do new file mode 100644 index 0000000..4e82c65 --- /dev/null +++ b/stata_code/data_extraction_processing/processing/annual/process_b2_haddock_OpenClose.do @@ -0,0 +1,94 @@ + + +/* in order to get the right distribution of lengths, you have to multiply the B2 length proportions by the number of B2's in each year. You have to multiply the A+B1 length proportions by the number of A+B1's in a year. + +Then you have to add these up. +*/ + + +/* process the b2 haddock */ +use "$my_outputdir/haddock_landings_OpenClose_$working_year.dta", clear +gen ab1=landings +keep year open ab1 +tempfile claim_harvest +sort year +save `claim_harvest' + +use "$my_outputdir/haddock_landings_OpenClose_$working_year.dta", clear +rename b2 release +keep year open release +tempfile cm +sort year +save `cm' + +use "$my_outputdir/haddock_b2_OpenClose_$working_year.dta", clear + + +sort year +merge m:1 year open using `cm' +replace release=0 if release==. +drop _merge + + +tempvar tt +bysort year open : egen `tt'=total(count) +gen prob=count/`tt' + +gen b2_count=prob*release + +keep year open l_in_bin b2_count + +sort year open l_in_bin +/* save */ +save "$my_outputdir/haddock_b2_counts_OpenClose_$working_year.dta", replace + + + + +use "$my_outputdir/haddock_ab1_OpenClose_$working_year.dta", clear + + +sort year + +cap drop _merge +merge m:1 year open using `claim_harvest' + + +tempvar ttt +bysort year open : egen `ttt'=total(count) +gen prob=count/`ttt' + +gen ab1_count=prob*ab1 + +keep year open l_in_bin ab1_count + +sort year open l_in_bin + +merge 1:1 year open l_in_bin using "$my_outputdir/haddock_b2_counts_OpenClose_$working_year.dta" +replace ab1_count=0 if ab1_count==. +replace b2_count=0 if b2_count==. +gen countnumbersoffish=round(ab1_count+b2_count) +tempfile tth1 +save `tth1' + +keep year open l_in_bin countnumbersoffish ab1_count b2_count + +rename l_in_bin lngcatinches +sort year open lngcatinches + +label var lngcatinches "length in inches" + +save "$my_outputdir/haddock_size_class_OpenClose_$working_year.dta", replace + + +/* compute the numbers of sub-legal fish retained +use `tth1', clear +keep year l_in_bin ab1_count countnumbers + +save "$my_outputdir/haddock_sublegals_annual_$working_year.dta", replace */ + + + + + + diff --git a/stata_code/data_extraction_processing/processing/batch_file_to_process_OpenClose_mrip_data.do b/stata_code/data_extraction_processing/processing/batch_file_to_process_OpenClose_mrip_data.do new file mode 100644 index 0000000..615d0a0 --- /dev/null +++ b/stata_code/data_extraction_processing/processing/batch_file_to_process_OpenClose_mrip_data.do @@ -0,0 +1,263 @@ +/* +This probably isn't doing everything we need it to do + +Need to read in the 2b95 landings. Doesn't loop over years + +we probably won' use this anymore + + + + +This is just a little helper file that calls all the yearly files and stacks the wave-level data into single datasets +It also aggregates everything into the proper format for the recreational bioeconomic model +It's quite awesome. + + +Running survey commands on multiple years takes a very long time. + +In order to do a data update, you will need to: + +1. run the copy_over_raw_mrip.do to copy and convert the sas7bdat files to dta. + +2. Run get_ma_allocation to get Recreation sites for MA (NORTH SOUTH) and + +3. Change the working year to the most recent year. + + CHECK for missing waves in the "ab1_lengths", catch totals, catch frequencies. + */ + + +global my_outputdir "${data_main}/MRIP_$vintage_string/annual" +capture mkdir "$my_outputdir" + +/*Set up the catchlist, triplist, and b2list global macros. These hold the filenames that are needed to figure out the catch, length-frequency, trips, and other things.*/ + + + +/********************************************************************************/ +/********************************************************************************/ +/* Use these to control the years and species for which the MRIP data is polled/queried*/ +/********************************************************************************/ +/********************************************************************************/ +global working_year 2019 +global working_year 2020 +global working_year 2021 +global working_year 2022 +global working_year 2023 + +local year $working_year +global wavelist 1 2 3 4 5 6 +global species1 "atlanticcod" +global species2 "haddock" +/*this is dumb, but I'm too lazy to replace everything that referred to these local/globals */ + + + +/*catchlist -- this assembles then names of files that are needed in the catchlist */ +/*Check to see if the file exists */ /* If the file exists, add the filename to the list if there are observations */ + + +global catchlist: dir "${data_raw}" files "catch_$working_year*.dta" +global triplist: dir "${data_raw}" files "trip_$working_year*.dta" +global b2list: dir "${data_raw}" files "size_b2_$working_year*.dta" +global sizelist: dir "${data_raw}" files "size_$working_year*.dta" + + +/* catch frequencies per trip*/ + +/* need to drop +id_code=="1757420210428005" from the sizelist, triplist, and catchlist. This is a trip that took place inshore in 2020 that caught 2 fish that were probably not cod + + +id_code="1792820210410001" should be dropped from the triplist catchlist when computing targeted trips. + +no changes to triplist or b2list + +The syntax will be to set up a global and then do +cap drop $drop_conditional after loading in the catchlist or sizelist files + + If we don't need to drop anything, the capture will swallow the error term. + And if we do, the code will drop the appropriate rows. +*/ + +global drop_conditional + + +/* catch totals -- these are done for all 3 years at once*/ + +global my_common "atlanticcod" + + + if "$my_common"=="atlanticcod"{ + global drop_conditional `"if id_code=="1757420210428005" "' + } + else { + global drop_conditional + } + +do "${processing_code}/annual/domain_cod_Open_Close_catch_totals.do" + +use "$my_outputdir/atlanticcod_catch_OpenClose_$working_year.dta", clear +keep year open tot_cat claim harvest release +rename claim a +rename harvest b1 +rename release b2 +rename tot_cat tot_catch +gen landings=a+b1 +save "$my_outputdir/atlanticcod_landings_OpenClose_$working_year.dta", replace + + + + + + + + + + + + + + +global my_common "haddock" + + if "$my_common"=="atlanticcod"{ + global drop_conditional `"if id_code=="1757420210428005" "' + } + else { + global drop_conditional + } + + +do "${processing_code}/annual/domain_haddock_Open_Close_catch_totals.do" + + +use "$my_outputdir/haddock_catch_OpenClose_$working_year.dta", clear +keep year open tot_cat claim harvest release +rename claim a +rename harvest b1 +rename release b2 +rename tot_cat tot_catch +gen landings=a+b1 +save "$my_outputdir/haddock_landings_OpenClose_$working_year.dta", replace + + + + + +clear + +/* length frequencies */ + + + +foreach sp in atlanticcod haddock{ + global my_common `sp' + + + if "$my_common"=="atlanticcod"{ + global drop_conditional `"if id_code=="1757420210428005" "' + } + else { + global drop_conditional + } + + + do "${processing_code}/annual/length_freqs_by_Open_Close_gom.do" + + clear +} + +/*stack together multiple, cleanup extras and delete */ +local cod_wave_ab1: dir "$my_outputdir" files "atlanticcodl_in_bin_a1b1_OpenClose*.dta" + +foreach file of local cod_wave_ab1{ + clear + append using ${my_outputdir}/`file' + ! rm ${my_outputdir}/`file' +} +save "$my_outputdir/cod_ab1_OpenClose_$working_year.dta", replace + + +/*stack together multiple, cleanup extras and delete */ +local haddock_wave_ab1: dir "$my_outputdir" files "haddockl_in_bin_a1b1_OpenClose*.dta" + +foreach file of local haddock_wave_ab1{ + clear + + append using ${my_outputdir}/`file' + ! rm ${my_outputdir}/`file' +} +save "$my_outputdir/haddock_ab1_OpenClose_$working_year.dta", replace + + +/* B2 length frequencies per wave*/ +foreach sp in atlanticcod haddock{ + global my_common `sp' + + + if "$my_common"=="atlanticcod"{ + global drop_conditional `"if id_code=="1757420210428005" "' + } + else { + global drop_conditional + } + + + do "${processing_code}/annual/b2_length_freqs_by_Open_Close_gom.do" + + +} + +/*stack these into a single dataset */ +clear +local cod_wave_b2: dir "$my_outputdir" files "atlanticcodl_in_bin_b2_OpenClose*.dta" +local haddock_wave_b2: dir "$my_outputdir" files "haddockl_in_bin_b2_OpenClose*.dta" + +clear +foreach file of local cod_wave_b2{ + + append using ${my_outputdir}/`file' + ! rm ${my_outputdir}/`file' +} + + +save "$my_outputdir/cod_b2_OpenClose_$working_year.dta", replace + clear +foreach file of local haddock_wave_b2{ + + append using ${my_outputdir}/`file' + ! rm ${my_outputdir}/`file' +} + +save "$my_outputdir/haddock_b2_OpenClose_$working_year.dta", replace + + + + + + + + + + + + + + + + + + +/*need to edit both of these to point to "annual" data files +I think the haddock one wokrs, just need to pull over the changes to the cod one. +*/ +/* join the b2 length data with the total number of released to get the length distribution for the number of fish released */ + +do "${processing_code}/annual/process_b2_haddock_OpenClose.do" +do "${processing_code}/annual/process_b2_cod_OpenClose.do" + + + + +global drop_conditional `"if id_code=="1792820210410001" "' diff --git a/stata_code/data_extraction_processing/processing/batch_file_to_process_annual_mrip_data.do b/stata_code/data_extraction_processing/processing/batch_file_to_process_annual_mrip_data.do index 9a4fb00..2ba3a07 100644 --- a/stata_code/data_extraction_processing/processing/batch_file_to_process_annual_mrip_data.do +++ b/stata_code/data_extraction_processing/processing/batch_file_to_process_annual_mrip_data.do @@ -43,6 +43,7 @@ global working_year 2019 global working_year 2020 global working_year 2021 global working_year 2022 +global working_year 2023 local year $working_year global wavelist 1 2 3 4 5 6 diff --git a/stata_code/data_extraction_processing/processing/catch_summaries.txt b/stata_code/data_extraction_processing/processing/catch_summaries.txt index 2a4d404..93fd671 100644 --- a/stata_code/data_extraction_processing/processing/catch_summaries.txt +++ b/stata_code/data_extraction_processing/processing/catch_summaries.txt @@ -16,10 +16,13 @@ if strmatch("$user","minyangWin"){ /* setup directories */ global code_dir "${project_dir}/stata_code" global working_data "${project_dir}/working_data" +global my_annualdir "${data_main}/MRIP_$vintage_string/annual" global output_dir "${project_dir}/output" global this_working_year 2023 global last_year = $this_working_year - 1 -global two_years = $this_working_year - 2 +global two_years = $this_working_year - 2 +global BLAST_DIR "${BLAST_root}/cod_haddock_fy2024/source_data/mrip" + dyndoc "$processing_code/catch_summaries.txt", saving($my_results/catch_summaries_${this_working_year}.html) replace ---> @@ -29,13 +32,47 @@ dyndoc "$processing_code/catch_summaries.txt", saving($my_results/catch_summarie <> /* If you dyndoc this right after running your MRIP data processing code, you can comment this out.*/ do $MRIP_BLAST -global vintage_string 2023_12_18 +global vintage_string 2024_01_02 global my_outputdir "${data_main}/MRIP_$vintage_string/monthly" - +global my_annualdir "${data_main}/MRIP_$vintage_string/annual" global FY_dir "${data_main}/MRIP_$vintage_string/fishing_years" +global BLAST_DIR "${BLAST_root}/cod_haddock_fy2024/source_data/mrip" local my_common1 "atlanticcod" local my_common2 "haddock" + + +/* Here are some parameters */ +global mt_to_kilo=1000 +global kilo_to_lbs=2.20462262 +global cm_to_inch=0.39370787 +global mortality_release=0.15 + + + +/* l-W is updated to be consistent with PDB +These are the length-weight relationships for Cod and Haddock +GOM Cod Formula: +Wlive (kg) = 0.000005132·L(fork cm)3.1625 (p < 0.0001, n=4890) + +http://www.nefsc.noaa.gov/publications/crd/crd0903/ +Haddock Weight length formula +Annual: Wlive (kg) = 0.000009298·L(fork cm)3.0205 (p < 0.0001, n=4890) +GROUNDFISH ASSESSMENT UPDATES 2012 page 181 + +Fork length and total length are equivalentfor haddock and haddock*/ + + +global cod_lwa 0.000005132 +global cod_lwb 3.1625 +global had_lwa 0.000009298 +global had_lwe 3.0205 +global lngcat_offset_cod 0.5 +global lngcat_offset_haddock 0.5 + + + + /*global this_working_year = 2023 */ global last_year = $this_working_year - 1 global two_years = $this_working_year - 2 @@ -56,7 +93,7 @@ global hadd_recACL2023=610 global cod_recACL2023=192 -global mrip_vintage "2023_12_18" +global mrip_vintage "2024_01_02" global hadd_recACL2024=759 global cod_recACL2024=192 @@ -483,7 +520,7 @@ list year month ab1mt b2mt b2dead_mt ab1_count b2_count ab1_lbs_per_fish b2_lbs_ <> ~~~~ - +These weights are computed using the monthly length distributions from MRIP. That data is a little thin. ~~~~ @@ -527,14 +564,42 @@ format ab1_count b2_count %10.0fc keep if year==$this_working_year <> -<> + +<> list year month ab1mt b2mt b2dead_mt ab1_count b2_count ab1_lbs_per_fish b2_lbs_per_fish, abbreviate(16) sum(ab1mt b2mt b2dead_mt ab1_count b2_count) <> ~~~~ +### These weights are computed using the length distribution of catch for the Open and Closed periods separately + +~~~~ +<> + +/* load in total catch */ +use "${my_annualdir}/atlanticcod_weights_OpenClose_${this_working_year}.dta", replace +list year month ab1mt b2mt b2dead_mt ab1_count b2_count, abbreviate(16) sum(ab1mt b2mt b2dead_mt ab1_count b2_count) +<> +~~~~ + + + + + + + + + + + + + + + + + Catch Statistics for Haddock =============================================================== @@ -707,6 +772,124 @@ list year month ab1mt b2mt b2dead_mt ab1_count b2_count ab1_lbs_per_fish b2_lbs_ ~~~~ + + + +### These weights are computed using the annual length distribution of catch + +~~~~ +<> + + + +/* load in total catch */ + + + +use "${BLAST_DIR}/haddock_size_class_ANNUAL${this_working_year}.dta", replace +/*lngcat fishing_year count month*/ +drop fishing_year + +bysort month: egen t=total(count) +gen prob=count/t +keep month lngcat prob + +tempfile lengths +save `lengths' + + + + +use ${data_main}/MRIP_${vintage_string}\monthly\haddock_catch_${this_working_year}.dta +gen landings=claim+harvest +keep year month landings release +destring month, replace +merge 1:m month using `lengths', keep(1 3) + + +gen count_release=prob*release +gen count_landings=prob*landings + + +collapse (sum) count_release count_landings, by(year month lngcat) +/* compute weights */ + + +/* +Spring (waves 1, 2, 3) small (<50cm, <=19inches) 0.321 +Spring (waves 1, 2, 3) large (>=50cm, >19inches) 0.113 +Fall (waves 4, 5, 6) small (<50cm, <=19inches) 0.742 +Fall (waves 4, 5, 6) large (>=50cm, >19inches) 0.459 +*/ + +gen discard_mortality=.113 if month<=6 & lngcat>19 +replace discard_mortality=.459 if month>6 & lngcat>19 + +replace discard_mortality=.321 if month<=6 & lngcat<=19 +replace discard_mortality=.742 if month>6 & lngcat<=19 + + + + +/* l_in_bin from MRIP is length of catch, rounded down to the nearest inch */ + +gen weight_per_fish= $kilo_to_lbs*$had_lwa*((lngcat+.5)/$cm_to_inch)^$had_lwe +gen landed_weight=weight_per_fish*count_landings +gen released_weight=weight_per_fish*count_release + +gen b2dead_weight=weight_per_fish*discard_mortality*count_release + +gen b2dead_count=discard_mortality*count_release + +collapse (sum) count_release count_landings landed_weight released_weight b2dead_count b2dead_weight, by(year month) + + +gen ab1_lbs_per_fish=landed_weight /count_landings +gen b2_lbs_per_fish=released_weight/count_release + + +replace landed_weight=landed_weight/($mt_to_kilo*$kilo_to_lbs) +replace released_weight=released_weight/($mt_to_kilo*$kilo_to_lbs) +replace b2dead_weight=b2dead_weight/($mt_to_kilo*$kilo_to_lbs) + + +rename landed_weight ab1mt +rename released_weight b2mt + +rename count_landings ab1_count +rename count_release b2_count + + +format ab1_lbs_per_fish b2_lbs_per_fish %4.3fc + +format ab1_count b2_count b2dead_count %10.0fc + +format b2mt ab1mt b2dead_weight %6.1fc + +<> +<> + +list year month ab1mt b2mt b2dead_weight ab1_count b2_count b2dead_count, abbreviate(16) sum(ab1mt b2mt b2dead_weight ab1_count b2_count b2dead_count) +<> +~~~~ + + + + + + + + + + + + + + + + + + Length distributions for haddock landed. Just the Fish in the 15-19" range. There were no B2's measured on Private boats, so this assumes the B2's on private boats is similar to the B2s on ForHire and Charter vessels. @@ -748,12 +931,10 @@ There were no B2's measured on Private boats, so I've set the length distributio <> -global mode_vintage 2023_11_30 - -global mode_dir "${data_main}/MRIP_$mode_vintage/mode" +global mode_dir "${data_main}/MRIP_${vintage_string}/mode" -use "${mode_dir}/haddock_size_mode_class_${working_year}.dta", clear -keep if year==$working_year +use "${mode_dir}/haddock_size_mode_class_${this_working_year}.dta", clear +keep if year==$this_working_year rename lngcat inches sort mode year month inches diff --git a/stata_code/data_extraction_processing/processing/convert_monthly_to_annual.do b/stata_code/data_extraction_processing/processing/convert_monthly_to_annual.do index b27f84a..2f6c845 100644 --- a/stata_code/data_extraction_processing/processing/convert_monthly_to_annual.do +++ b/stata_code/data_extraction_processing/processing/convert_monthly_to_annual.do @@ -47,17 +47,19 @@ save "${BLAST_DIR}/cod_catch_class_ANNUAL`yr1'.dta", replace replace count=count/1000 label var count "count 1000s" xtline count +xtline count if month==9 , subtitle("Averaged across all months") graph export "${my_images_vintage}/cod_catch_class_ANNUAL`yr1'.tif", as(tif) replace -xtline count if num_fish>0 +xtline count if month==9 & num_fish>0, subtitle("Averaged across all months, no zeros") graph export "${my_images_vintage}/cod_catch_class_no0_ANNUAL`yr1'.tif", as(tif) replace bysort month: egen tc=total(count) gen prob=count/tc -xtline prob +xtline prob if month==9 , subtitle("Averaged across all months") + graph export "${my_images_vintage}/cod_catch_classP_ANNUAL`yr1'.tif", as(tif) replace -xtline prob if num_fish>0 +xtline prob if month==9 & num_fish>0, subtitle("Averaged across all months, no zeros") graph export "${my_images_vintage}/cod_catch_classPno0_ANNUAL`yr1'.tif", as(tif) replace @@ -98,18 +100,21 @@ save "${BLAST_DIR}/haddock_catch_class_ANNUAL`yr1'.dta", replace replace count=count/1000 label var count "count 1000s" xtline count +xtline count if month==9 , subtitle("Averaged across all months") + graph export "${my_images_vintage}/haddock_catch_class_ANNUAL`yr1'.tif", as(tif) replace -xtline count if num_fish>0 +xtline count if month==9 & num_fish>0, subtitle("Averaged across all months, no zeros") graph export "${my_images_vintage}/haddock_catch_class_no0_ANNUAL`yr1'.tif", as(tif) replace bysort month: egen tc=total(count) gen prob=count/tc -xtline prob +xtline prob if month==9 , subtitle("Averaged across all months") graph export "${my_images_vintage}/haddock_catch_classP_ANNUAL`yr1'.tif", as(tif) replace -xtline prob if num_fish>0 +xtline prob if month==9 & num_fish>0, subtitle("Averaged across all months, no zeros") + graph export "${my_images_vintage}/haddock_catch_classPno0_ANNUAL`yr1'.tif", as(tif) replace /*******************END HADDOCK **************************************/ @@ -166,14 +171,14 @@ save "${BLAST_DIR}/cod_size_class_ANNUAL`yr1'.dta", replace replace count=count/1000 label var count "count 1000s" -xtline count, xline($cmin) +xtline count if month==9, xline($cmin) subtitle("Averaged across all months") graph export "${my_images_vintage}/cod_size_class_ANNUAL`yr1'.tif", as(tif) replace bysort month: egen tc=total(count) gen prob=count/tc -xtline prob, xline($cmin) +xtline prob if month==9, xline($cmin) subtitle("Averaged across all months") graph export "${my_images_vintage}/cod_size_classP_ANNUAL`yr1'.tif", as(tif) replace @@ -188,27 +193,26 @@ keep if fishing_year==`yr1' | (fishing_year==`lastyr' & inlist(month,1,2,3,4,11, replace fishing_year=`yr1' -gen open=inlist(month,4,9,10) +gen open=inlist(month,9,10) collapse (sum) count, by(lngcat fishing_year open) preserve keep if open==1 -expand 3, gen(o1) +expand 2, gen(o1) sort fishing_year lngcat -gen month=4 if o1==0 +gen month=9 if o1==0 sort fishing_year lngcat o1 -bysort fishing_year lngcat: replace month=_n+7 if o1==1 +bysort fishing_year lngcat: replace month=_n+8 if o1==1 tempfile open drop o1 save `open', replace restore keep if open==0 -expand 9, gen(cl) +expand 10, gen(cl) sort fishing_year lngcat cl bysort fishing_year lngcat: gen month=_n -bysort fishing_year lngcat: replace month=_n+1 if month>=4 -bysort fishing_year lngcat: replace month=_n+3 if month>=9 +bysort fishing_year lngcat: replace month=_n+2 if month>=9 append using `open' @@ -221,18 +225,25 @@ save "${BLAST_DIR}/cod_size_class_OPEN_SPLIT`yr1'.dta", replace xtset month lng +bysort month: egen tc=total(count) replace count=count/1000 label var count "count 1000s" -xtline count, xline($cmin) -graph export "${my_images_vintage}/cod_size_class_OPEN_SPLIT`yr1'.tif", as(tif) replace +gen open="open" +replace open="closed" if month==8 +xtline count if inlist(month,8,9), xline($cmin) i(open) t(lng) + +graph export "${my_images_vintage}/cod_size_class_OPEN_SPLIT`yr1'.tif", as(tif) replace +cap drop tc bysort month: egen tc=total(count) gen prob=count/tc -xtline prob, xline($cmin) + +xtline prob if inlist(month,8,9), xline($cmin) i(open) t(lng) + graph export "${my_images_vintage}/cod_size_classP_OPEN_SPLIT`yr1'.tif", as(tif) replace @@ -270,13 +281,13 @@ save "${BLAST_DIR}/haddock_size_class_ANNUAL`yr1'.dta", replace replace count=count/1000 label var count "count 1000s" -xtline count, xline($hmin) +xtline count if month==9, xline($hmin) subtitle("Averaged across all months") graph export "${my_images_vintage}/haddock_size_class_ANNUAL`yr1'.tif", as(tif) replace bysort month: egen tc=total(count) gen prob=count/tc -xtline prob, xline($hmin) +xtline prob if month==9, xline($hmin) subtitle("Averaged across all months") graph export "${my_images_vintage}/haddock_size_classP_ANNUAL`yr1'.tif", as(tif) replace diff --git a/stata_code/data_extraction_processing/processing/monthly/cod_monthly_weight_ab1b2.do b/stata_code/data_extraction_processing/processing/monthly/cod_monthly_weight_ab1b2.do index f19f708..429f56b 100644 --- a/stata_code/data_extraction_processing/processing/monthly/cod_monthly_weight_ab1b2.do +++ b/stata_code/data_extraction_processing/processing/monthly/cod_monthly_weight_ab1b2.do @@ -39,9 +39,9 @@ gen weight_per_fish= $kilotolb*$coda*((l_in_bin+.5)/$cmtoinch)^$codb -gen ab1weight=round(ab1_count)*weight_per_fish -gen b2weight=round(b2_count)*weight_per_fish -gen b2weight_dead=round(b2_count)*weight_per_fish*discard_mortality +gen ab1weight=ab1_count*weight_per_fish +gen b2weight=b2_count*weight_per_fish +gen b2weight_dead=b2_count*weight_per_fish*discard_mortality keep if year==$working_year diff --git a/stata_code/data_extraction_processing/processing/monthly/haddock_monthly_weight_ab1b2.do b/stata_code/data_extraction_processing/processing/monthly/haddock_monthly_weight_ab1b2.do index cff4a98..8df5eb6 100644 --- a/stata_code/data_extraction_processing/processing/monthly/haddock_monthly_weight_ab1b2.do +++ b/stata_code/data_extraction_processing/processing/monthly/haddock_monthly_weight_ab1b2.do @@ -1,10 +1,20 @@ use "${data_main}/MRIP_${vintage_string}/monthly/haddock_b2_counts_${working_year}.dta", clear -gen discard_mortality=.113 if month<=6 & l_in_bin>=20 -replace discard_mortality=.459 if month>6 & l_in_bin>=20 +gen discard_mortality=.113 if month<=6 & l_in_bin>19 +replace discard_mortality=.459 if month>6 & l_in_bin>19 -replace discard_mortality=.321 if month<=6 & l_in_bin<20 -replace discard_mortality=.742 if month>6 & l_in_bin<20 +replace discard_mortality=.321 if month<=6 & l_in_bin<=19 +replace discard_mortality=.742 if month>6 & l_in_bin<=19 + + + + +/* +Spring (waves 1, 2, 3) small (<50cm, <=19inches) 0.321 +Spring (waves 1, 2, 3) large (>=50cm, >19inches) 0.113 +Fall (waves 4, 5, 6) small (<50cm, <=19inches) 0.742 +Fall (waves 4, 5, 6) large (>=50cm, >19inches) 0.459 +*/ tempfile b2_adj save `b2_adj', replace @@ -45,14 +55,14 @@ drop if l_in_bin==0 gen weight_per_fish= $kilotolb*$hada*((l_in_bin+.5)/$cmtoinch)^$hade -gen ab1weight=round(ab1_count)*weight_per_fish -gen b2weight=round(b2_count)*weight_per_fish +gen ab1weight=ab1_count*weight_per_fish +gen b2weight=b2_count*weight_per_fish keep if year==$working_year drop if l_in_bin==0 -gen b2weight_dead=round(b2_count)*weight_per_fish*discard_mortality +gen b2weight_dead=b2_count*weight_per_fish*discard_mortality collapse (sum) ab1weight b2weight b2weight_dead ab1_count b2_count , by(year month) diff --git a/stata_code/data_extraction_processing/processing/monthly/process_b2_cod.do b/stata_code/data_extraction_processing/processing/monthly/process_b2_cod.do index 67d5ee1..3eab3f0 100644 --- a/stata_code/data_extraction_processing/processing/monthly/process_b2_cod.do +++ b/stata_code/data_extraction_processing/processing/monthly/process_b2_cod.do @@ -91,22 +91,12 @@ save "$my_outputdir/atlanticcod_b2_counts_$working_year.dta", replace use "$my_outputdir/cod_ab1_$working_year.dta", clear -/* -expand 2 if month==5 & year==2013 -drop if month==6 & year==2013 - -bysort year month l_in_bin: gen mark=_n -replace month=6 if mark==2 & year==2013 & month==5 - -drop mark -*/ - sort year month cap drop _merge merge m:1 year month using `claim_harvest' - +drop if l_in_bin==. tempvar ttt bysort year month: egen `ttt'=total(count) gen prob=count/`ttt' diff --git a/stata_code/data_extraction_processing/processing/process_wrapper.do b/stata_code/data_extraction_processing/processing/process_wrapper.do index 56cc0af..500a424 100644 --- a/stata_code/data_extraction_processing/processing/process_wrapper.do +++ b/stata_code/data_extraction_processing/processing/process_wrapper.do @@ -30,9 +30,7 @@ global BLAST_DIR "${BLAST_root}/cod_haddock_fy2024/source_data/mrip" -/*this computes calendar year estimates, which you don't really use -do "$processing_code/batch_file_to_process_annual_mrip_data.do" - */ + /*this computes monthly estimates*/ @@ -66,6 +64,39 @@ do "$processing_code/monthly/joint_catch_frequencies.do" /* Write to a html file.*/ + + +/*this computes calendar year estimates, which you don't really use */ +do "$processing_code/batch_file_to_process_annual_mrip_data.do" + + + + +/*this computes calendar year estimates, which you don't really use */ +do "$processing_code/batch_file_to_process_OpenClose_mrip_data.do" + + + + +/*apply Open/Close distributions to the monthly catches */ + + + +do "$processing_code/annual/cod_weights_using_OpenClose.do" + + + + + + + + + + + + + + global this_working_year 2023 dyndoc "$processing_code/catch_summaries.txt", saving($my_results/catch_summaries_${this_working_year}.html) replace