From 8f544bf774420c33810cca2337994ad798dd1958 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Wed, 31 Jul 2024 15:29:20 +0200 Subject: [PATCH 01/73] ADD script for validation plots for standalone ARC detector --- scripts/ARC_val_plots.py | 260 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 scripts/ARC_val_plots.py diff --git a/scripts/ARC_val_plots.py b/scripts/ARC_val_plots.py new file mode 100644 index 0000000..8996d61 --- /dev/null +++ b/scripts/ARC_val_plots.py @@ -0,0 +1,260 @@ +import ROOT +from podio import root_io +import dd4hep as dd4hepModule +from ROOT import dd4hep +import matplotlib.pyplot as plt +from matplotlib.ticker import MultipleLocator, FuncFormatter +from matplotlib.colors import SymLogNorm + +import argparse +import numpy as np +import os +from dd4hep import Detector +import DDRec + + + +def format_func(value, tick_number): + N = int(np.round(value / np.pi * 6)) + if N == 0: + return "0" + elif N % 6 == 0: + if N == 6: + return r"$\pi$" + elif N == -6: + return r"$-\pi$" + else: + return r"${0}\pi$".format(N // 6) + elif N % 3 == 0: + if N == 3: + return r"$\pi/2$" + elif N == -3: + return r"$-\pi/2$" + else: + return r"${0}\pi/2$".format(N // 3) + elif N % 2 == 0: + if N == 2: + return r"$\pi/3$" + elif N == -2: + return r"$-\pi/3$" + else: + return r"${0}\pi/3$".format(N // 2) + else: + if N == 1: + return r"$\pi/6$" + elif N == -1: + return r"-$\pi/6$" + else: + return r"${0}\pi/6$".format(N) + + +def count_photons(inputFile): + + # set file reader + podio_reader = root_io.Reader(inputFile) + + # get detector description for cell id decoding + theDetector = Detector.getInstance() + theDetector.fromXML("../Detector_Description/ARC/ARC_standalone_o1_v01.xml") + idposConv = DDRec.CellIDPositionConverter(theDetector) + n_evts = len(podio_reader.get("events")) + + + ########## Count Photon Hits ######################### + + # prepare arrays for quantities of interest + n_ph = np.zeros(n_evts) + first_hit = np.zeros((2, n_evts)) + ph_theta = [] + ph_phi = [] + + # loop over dataset + for i_evt, event in enumerate(podio_reader.get("events")): + + p = (event.get("MCParticles"))[0] + mom = ROOT.TVector3(p.getMomentum().x, p.getMomentum().y, p.getMomentum().z) + first_hit[0,i_evt] = mom.Theta() + first_hit[1,i_evt] = mom.Phi() + + for i_hit, arc_hit in enumerate(event.get("ArcCollection")): + particle = arc_hit.getMCParticle() + pdgID = particle.getPDG() + if pdgID == 22 or pdgID == -22: + n_ph[i_evt] += 1 + cellID = arc_hit.getCellID() + x = idposConv.position(cellID) + ph_theta.append(x.theta()) + ph_phi.append(x.phi()) + + return n_ph, first_hit, [ph_theta, ph_phi] + + +def plot_ph_count(n_ph, first_hit, ph_count, args): + + n_evt = len(n_ph) + print("Number of evts:", n_evt) + + filename = os.path.splitext(os.path.basename(args.inputFile))[0] # removes path and extension + if args.outputPath[-1] != '/': + args.outputPath += '/' + output_start = args.outputPath + filename + + + # photon count vs. theta + + fig1, ax1 = plt.subplots(figsize=(8,6)) + nbins=90 + hist, bin_edges = np.histogram(ph_count[0], bins=nbins, range=(0, np.pi)) + if not args.no_norm: + hist = hist.astype(float)/n_evt + ax1.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), + color='lightblue', edgecolor='black', align='edge') + ax1.set_xlabel(r'Polar angle $\theta$') + ax1.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') + ax1.set_yscale('log') + ax1.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax1.xaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig1.savefig(output_start+'_phCount_theta.png', bbox_inches='tight') + if args.no_show: + plt.close(fig1) + + + # photon count vs. theta and phi + + fig2, ax2 = plt.subplots(figsize=(8,6)) + nbins_theta = 90 + nbins_phi = 36 + hist, x_edges, y_edges = np.histogram2d(ph_count[0], ph_count[1], + bins=[nbins_theta, nbins_phi], + range=((0, np.pi), (-np.pi, np.pi))) + vmax=1e4 + lim=1 + if not args.no_norm: + hist = hist.astype(float)/n_evt + vmax/=n_evt + lim/=n_evt + img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], + aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) + cbar = plt.colorbar(img, ax=ax2) + cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') + + ax2.set_xlabel(r'Polar angle $\theta$') + ax2.set_ylabel(r'Azimuthal angle $\phi$') + ax2.set_title('Photon Count') + ax2.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax2.xaxis.set_major_formatter(FuncFormatter(format_func)) + ax2.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax2.yaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig2.savefig(output_start+'_phCount_thetaphi.png', bbox_inches='tight') + if args.no_show: + plt.close(fig2) + + + # number of photons per evt + + fig3, ax3 = plt.subplots(figsize=(8,6)) + nbins=50 + ax3.hist(n_ph, bins=nbins, range=(0, 250), + color='lightblue', edgecolor='black') + ax3.set_xlabel('Number of photons') + ax3.set_ylabel(f'Photon count / 5') + ax3.set_yscale('log') + + if not args.no_save: + fig3.savefig(output_start+'_phCount_Nph.png', bbox_inches='tight') + if args.no_show: + plt.close(fig3) + + + # photon count vs. theta of first hit + + fig4, ax4 = plt.subplots(figsize=(8,6)) + nbins=90 + hist, bin_edges = np.histogram(first_hit[0], bins=nbins, range=(0, np.pi), weights=n_ph) + if not args.no_norm: + hist = hist.astype(float)/n_evt + ax4.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), + color='lightblue', edgecolor='black', align='edge') + ax4.set_xlabel(r'Polar angle $\theta$ of primary particle') + ax4.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') + ax4.set_yscale('log') + ax4.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax4.xaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig4.savefig(output_start+'_phCount_theta_1stHit.png', bbox_inches='tight') + if args.no_show: + plt.close(fig4) + + + # photon count vs. theta and phi of first hit + + fig5, ax5 = plt.subplots(figsize=(8,6)) + nbins_theta = 90 + nbins_phi = 36 + hist, x_edges, y_edges = np.histogram2d(first_hit[0], first_hit[1], + bins=[nbins_theta, nbins_phi], + range=((0, np.pi), (-np.pi, np.pi)), + weights=n_ph) + vmax=1e4 + lim=1 + if not args.no_norm: + hist = hist.astype(float)/n_evt + vmax/=n_evt + lim/=n_evt + img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], + aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) + cbar = plt.colorbar(img, ax=ax5) + cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') + + ax5.set_xlabel(r'Polar angle $\theta$ of primary particle') + ax5.set_ylabel(r'Azimuthal angle $\phi$ of primary particle') + ax5.set_title('Photon Count') + ax5.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax5.xaxis.set_major_formatter(FuncFormatter(format_func)) + ax5.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax5.yaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig5.savefig(output_start+'_phCount_thetaphi_1stHit.png', bbox_inches='tight') + if args.no_show: + plt.close(fig5) + + + else: + plt.show() + + +######################################################################### + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process simulation" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the simulation file to be processed', default='arc_pi+_50GeV.root') + parser.add_argument('-o', "--outputPath", type=str, + help='The name of the directory where to save output files', default='./') + parser.add_argument("--no_save", action='store_true', help='Do not save output arrays') + parser.add_argument("--no_show", action='store_true', help='Do not plot output histograms') + parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') + + args = parser.parse_args() + + n_ph, first_hit, ph_count = count_photons(args.inputFile) + ph_presence = (len(ph_count[0]) != 0) + + if not(args.no_save and args.no_show): + if ph_presence: + plot_ph_count(n_ph, first_hit, ph_count, args) + + print(ph_presence) + + + + + From 8f8ea4e7889673e993f14b009dcf01c3cdc3eacc Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 5 Aug 2024 16:13:52 +0200 Subject: [PATCH 02/73] ADD structure and new scripts for validation pipeline --- .../CLD/CLD_o3_v01/analysis/ARC_val_plots.py | 268 ++++++++++++++++++ .../ARC_standalone_o1_v01_plot.sh | 16 ++ .../ARC_standalone_o1_v01_setup.sh | 8 + .../ARC_standalone_o1_v01_simulation.sh | 13 + 4 files changed, 305 insertions(+) create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py new file mode 100644 index 0000000..923c4c1 --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py @@ -0,0 +1,268 @@ +import ROOT +from podio import root_io +import dd4hep as dd4hepModule +from ROOT import dd4hep +import matplotlib.pyplot as plt +from matplotlib.ticker import MultipleLocator, FuncFormatter +from matplotlib.colors import SymLogNorm + +import argparse +import numpy as np +import os +from dd4hep import Detector +import DDRec + + + +def find_directory(directory_name, search_path): + for root, dirs, files in os.walk(search_path): + if directory_name in dirs: + return os.path.join(root, directory_name) + return None + + +def format_func(value, tick_number): + N = int(np.round(value / np.pi * 6)) + if N == 0: + return "0" + elif N % 6 == 0: + if N == 6: + return r"$\pi$" + elif N == -6: + return r"$-\pi$" + else: + return r"${0}\pi$".format(N // 6) + elif N % 3 == 0: + if N == 3: + return r"$\pi/2$" + elif N == -3: + return r"$-\pi/2$" + else: + return r"${0}\pi/2$".format(N // 3) + elif N % 2 == 0: + if N == 2: + return r"$\pi/3$" + elif N == -2: + return r"$-\pi/3$" + else: + return r"${0}\pi/3$".format(N // 2) + else: + if N == 1: + return r"$\pi/6$" + elif N == -1: + return r"-$\pi/6$" + else: + return r"${0}\pi/6$".format(N) + + +def count_photons(inputFile): + + # set file reader + podio_reader = root_io.Reader(inputFile) + + # get detector description for cell id decoding + k4geo_path = find_directory("k4geo", "/validation") + theDetector = Detector.getInstance() + theDetector.fromXML(k4geo_path+"/test/compact/ARC_standalone_o1_v01.xml") + idposConv = DDRec.CellIDPositionConverter(theDetector) + n_evts = len(podio_reader.get("events")) + + + ########## Count Photon Hits ######################### + + # prepare arrays for quantities of interest + n_ph = np.zeros(n_evts) + first_hit = np.zeros((2, n_evts)) + ph_theta = [] + ph_phi = [] + + # loop over dataset + for i_evt, event in enumerate(podio_reader.get("events")): + + p = (event.get("MCParticles"))[0] + mom = ROOT.TVector3(p.getMomentum().x, p.getMomentum().y, p.getMomentum().z) + first_hit[0,i_evt] = mom.Theta() + first_hit[1,i_evt] = mom.Phi() + + for i_hit, arc_hit in enumerate(event.get("ArcCollection")): + particle = arc_hit.getMCParticle() + pdgID = particle.getPDG() + if pdgID == 22 or pdgID == -22: + n_ph[i_evt] += 1 + cellID = arc_hit.getCellID() + x = idposConv.position(cellID) + ph_theta.append(x.theta()) + ph_phi.append(x.phi()) + + return n_ph, first_hit, [ph_theta, ph_phi] + + +def plot_ph_count(n_ph, first_hit, ph_count, args): + + n_evt = len(n_ph) + print("Number of evts:", n_evt) + + filename = os.path.splitext(os.path.basename(args.inputFile))[0] # removes path and extension + if args.outputPath[-1] != '/': + args.outputPath += '/' + output_start = args.outputPath + filename + + + # photon count vs. theta + + fig1, ax1 = plt.subplots(figsize=(8,6)) + nbins=90 + hist, bin_edges = np.histogram(ph_count[0], bins=nbins, range=(0, np.pi)) + if not args.no_norm: + hist = hist.astype(float)/n_evt + ax1.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), + color='lightblue', edgecolor='black', align='edge') + ax1.set_xlabel(r'Polar angle $\theta$') + ax1.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') + ax1.set_yscale('log') + ax1.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax1.xaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig1.savefig(output_start+'_phCount_theta.svg', bbox_inches='tight') + if args.no_show: + plt.close(fig1) + + + # photon count vs. theta and phi + + fig2, ax2 = plt.subplots(figsize=(8,6)) + nbins_theta = 90 + nbins_phi = 36 + hist, x_edges, y_edges = np.histogram2d(ph_count[0], ph_count[1], + bins=[nbins_theta, nbins_phi], + range=((0, np.pi), (-np.pi, np.pi))) + vmax=1e4 + lim=1 + if not args.no_norm: + hist = hist.astype(float)/n_evt + vmax/=n_evt + lim/=n_evt + img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], + aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) + cbar = plt.colorbar(img, ax=ax2) + cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') + + ax2.set_xlabel(r'Polar angle $\theta$') + ax2.set_ylabel(r'Azimuthal angle $\phi$') + ax2.set_title('Photon Count') + ax2.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax2.xaxis.set_major_formatter(FuncFormatter(format_func)) + ax2.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax2.yaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig2.savefig(output_start+'_phCount_thetaphi.svg', bbox_inches='tight') + if args.no_show: + plt.close(fig2) + + + # number of photons per evt + + fig3, ax3 = plt.subplots(figsize=(8,6)) + nbins=50 + ax3.hist(n_ph, bins=nbins, range=(0, 250), + color='lightblue', edgecolor='black') + ax3.set_xlabel('Number of photons') + ax3.set_ylabel(f'Photon count / 5') + ax3.set_yscale('log') + + if not args.no_save: + fig3.savefig(output_start+'_phCount_Nph.svg', bbox_inches='tight') + if args.no_show: + plt.close(fig3) + + + # photon count vs. theta of first hit + + fig4, ax4 = plt.subplots(figsize=(8,6)) + nbins=90 + hist, bin_edges = np.histogram(first_hit[0], bins=nbins, range=(0, np.pi), weights=n_ph) + if not args.no_norm: + hist = hist.astype(float)/n_evt + ax4.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), + color='lightblue', edgecolor='black', align='edge') + ax4.set_xlabel(r'Polar angle $\theta$ of primary particle') + ax4.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') + ax4.set_yscale('log') + ax4.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax4.xaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig4.savefig(output_start+'_phCount_theta_1stHit.svg', bbox_inches='tight') + if args.no_show: + plt.close(fig4) + + + # photon count vs. theta and phi of first hit + + fig5, ax5 = plt.subplots(figsize=(8,6)) + nbins_theta = 90 + nbins_phi = 36 + hist, x_edges, y_edges = np.histogram2d(first_hit[0], first_hit[1], + bins=[nbins_theta, nbins_phi], + range=((0, np.pi), (-np.pi, np.pi)), + weights=n_ph) + vmax=1e4 + lim=1 + if not args.no_norm: + hist = hist.astype(float)/n_evt + vmax/=n_evt + lim/=n_evt + img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], + aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) + cbar = plt.colorbar(img, ax=ax5) + cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') + + ax5.set_xlabel(r'Polar angle $\theta$ of primary particle') + ax5.set_ylabel(r'Azimuthal angle $\phi$ of primary particle') + ax5.set_title('Photon Count') + ax5.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax5.xaxis.set_major_formatter(FuncFormatter(format_func)) + ax5.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax5.yaxis.set_major_formatter(FuncFormatter(format_func)) + + if not args.no_save: + fig5.savefig(output_start+'_phCount_thetaphi_1stHit.svg', bbox_inches='tight') + if args.no_show: + plt.close(fig5) + + + else: + plt.show() + + +######################################################################### + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process simulation" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the simulation file to be processed', default='arc_pi+_50GeV.root') + parser.add_argument('-o', "--outputPath", type=str, + help='The name of the directory where to save output files', default='./') + parser.add_argument("--no_save", action='store_true', help='Do not save output arrays') + parser.add_argument("--no_show", action='store_true', help='Do not plot output histograms') + parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') + + args = parser.parse_args() + + n_ph, first_hit, ph_count = count_photons(args.inputFile) + ph_presence = (len(ph_count[0]) != 0) + + if not(args.no_save and args.no_show): + if ph_presence: + plot_ph_count(n_ph, first_hit, ph_count, args) + + print(ph_presence) + + + + + diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh new file mode 100644 index 0000000..e295fd1 --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh @@ -0,0 +1,16 @@ +# make plots +echo "Sourcing key4hep..." +source /cvmfs/sw.hsf.org/key4hep/setup.sh +echo "Sourcing executed successfully" + +echo "Starting analysis and plot script..." +python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ + -f ARC_sim.root --no_show \ + -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots +echo "Script executed successfully" + +# upload them on website +echo "Starting website script..." +cd key4hep-reco-validation/web/python +python3 make_web.py --dest $WORKAREA/$PLOTAREA +echo "Script executed successfully" \ No newline at end of file diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh new file mode 100644 index 0000000..9f7f00d --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh @@ -0,0 +1,8 @@ +# setup phase + +echo "Downloading necessary Github repos..." + +git clone --depth 1 https://github.com/key4hep/k4geo.git +git clone https://github.com/key4hep/CLDConfig.git + +echo "Download terminated - setup stage successful" \ No newline at end of file diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh new file mode 100644 index 0000000..b511cc6 --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh @@ -0,0 +1,13 @@ +# simulation phase +echo "Sourcing key4hep..." +source /cvmfs/sw.hsf.org/key4hep/setup.sh +echo "Sourcing executed successfully" + +echo "Starting simulation..." +ddsim --steeringFile CLDConfig/CLDConfig/cld_arc__steer.py \ + --enableGun --gun.distribution cos(theta) \ + --gun.energy "20*GeV" --gun.particle proton \ + --numberOfEvents $NUMBER_OF_EVENTS \ + --outputFile ARC_sim.root \ + --random.enableEventSeed --random.seed 42 +echo "Simulation ended successfully" \ No newline at end of file From b5f0062c66c580a8e197674810f43dda9127532f Mon Sep 17 00:00:00 2001 From: Enrico Lupi <115542144+enlupi@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:25:00 +0200 Subject: [PATCH 03/73] Delete scripts/ARC_val_plots.py --- scripts/ARC_val_plots.py | 260 --------------------------------------- 1 file changed, 260 deletions(-) delete mode 100644 scripts/ARC_val_plots.py diff --git a/scripts/ARC_val_plots.py b/scripts/ARC_val_plots.py deleted file mode 100644 index 8996d61..0000000 --- a/scripts/ARC_val_plots.py +++ /dev/null @@ -1,260 +0,0 @@ -import ROOT -from podio import root_io -import dd4hep as dd4hepModule -from ROOT import dd4hep -import matplotlib.pyplot as plt -from matplotlib.ticker import MultipleLocator, FuncFormatter -from matplotlib.colors import SymLogNorm - -import argparse -import numpy as np -import os -from dd4hep import Detector -import DDRec - - - -def format_func(value, tick_number): - N = int(np.round(value / np.pi * 6)) - if N == 0: - return "0" - elif N % 6 == 0: - if N == 6: - return r"$\pi$" - elif N == -6: - return r"$-\pi$" - else: - return r"${0}\pi$".format(N // 6) - elif N % 3 == 0: - if N == 3: - return r"$\pi/2$" - elif N == -3: - return r"$-\pi/2$" - else: - return r"${0}\pi/2$".format(N // 3) - elif N % 2 == 0: - if N == 2: - return r"$\pi/3$" - elif N == -2: - return r"$-\pi/3$" - else: - return r"${0}\pi/3$".format(N // 2) - else: - if N == 1: - return r"$\pi/6$" - elif N == -1: - return r"-$\pi/6$" - else: - return r"${0}\pi/6$".format(N) - - -def count_photons(inputFile): - - # set file reader - podio_reader = root_io.Reader(inputFile) - - # get detector description for cell id decoding - theDetector = Detector.getInstance() - theDetector.fromXML("../Detector_Description/ARC/ARC_standalone_o1_v01.xml") - idposConv = DDRec.CellIDPositionConverter(theDetector) - n_evts = len(podio_reader.get("events")) - - - ########## Count Photon Hits ######################### - - # prepare arrays for quantities of interest - n_ph = np.zeros(n_evts) - first_hit = np.zeros((2, n_evts)) - ph_theta = [] - ph_phi = [] - - # loop over dataset - for i_evt, event in enumerate(podio_reader.get("events")): - - p = (event.get("MCParticles"))[0] - mom = ROOT.TVector3(p.getMomentum().x, p.getMomentum().y, p.getMomentum().z) - first_hit[0,i_evt] = mom.Theta() - first_hit[1,i_evt] = mom.Phi() - - for i_hit, arc_hit in enumerate(event.get("ArcCollection")): - particle = arc_hit.getMCParticle() - pdgID = particle.getPDG() - if pdgID == 22 or pdgID == -22: - n_ph[i_evt] += 1 - cellID = arc_hit.getCellID() - x = idposConv.position(cellID) - ph_theta.append(x.theta()) - ph_phi.append(x.phi()) - - return n_ph, first_hit, [ph_theta, ph_phi] - - -def plot_ph_count(n_ph, first_hit, ph_count, args): - - n_evt = len(n_ph) - print("Number of evts:", n_evt) - - filename = os.path.splitext(os.path.basename(args.inputFile))[0] # removes path and extension - if args.outputPath[-1] != '/': - args.outputPath += '/' - output_start = args.outputPath + filename - - - # photon count vs. theta - - fig1, ax1 = plt.subplots(figsize=(8,6)) - nbins=90 - hist, bin_edges = np.histogram(ph_count[0], bins=nbins, range=(0, np.pi)) - if not args.no_norm: - hist = hist.astype(float)/n_evt - ax1.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), - color='lightblue', edgecolor='black', align='edge') - ax1.set_xlabel(r'Polar angle $\theta$') - ax1.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') - ax1.set_yscale('log') - ax1.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax1.xaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig1.savefig(output_start+'_phCount_theta.png', bbox_inches='tight') - if args.no_show: - plt.close(fig1) - - - # photon count vs. theta and phi - - fig2, ax2 = plt.subplots(figsize=(8,6)) - nbins_theta = 90 - nbins_phi = 36 - hist, x_edges, y_edges = np.histogram2d(ph_count[0], ph_count[1], - bins=[nbins_theta, nbins_phi], - range=((0, np.pi), (-np.pi, np.pi))) - vmax=1e4 - lim=1 - if not args.no_norm: - hist = hist.astype(float)/n_evt - vmax/=n_evt - lim/=n_evt - img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], - aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) - cbar = plt.colorbar(img, ax=ax2) - cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') - - ax2.set_xlabel(r'Polar angle $\theta$') - ax2.set_ylabel(r'Azimuthal angle $\phi$') - ax2.set_title('Photon Count') - ax2.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax2.xaxis.set_major_formatter(FuncFormatter(format_func)) - ax2.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax2.yaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig2.savefig(output_start+'_phCount_thetaphi.png', bbox_inches='tight') - if args.no_show: - plt.close(fig2) - - - # number of photons per evt - - fig3, ax3 = plt.subplots(figsize=(8,6)) - nbins=50 - ax3.hist(n_ph, bins=nbins, range=(0, 250), - color='lightblue', edgecolor='black') - ax3.set_xlabel('Number of photons') - ax3.set_ylabel(f'Photon count / 5') - ax3.set_yscale('log') - - if not args.no_save: - fig3.savefig(output_start+'_phCount_Nph.png', bbox_inches='tight') - if args.no_show: - plt.close(fig3) - - - # photon count vs. theta of first hit - - fig4, ax4 = plt.subplots(figsize=(8,6)) - nbins=90 - hist, bin_edges = np.histogram(first_hit[0], bins=nbins, range=(0, np.pi), weights=n_ph) - if not args.no_norm: - hist = hist.astype(float)/n_evt - ax4.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), - color='lightblue', edgecolor='black', align='edge') - ax4.set_xlabel(r'Polar angle $\theta$ of primary particle') - ax4.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') - ax4.set_yscale('log') - ax4.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax4.xaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig4.savefig(output_start+'_phCount_theta_1stHit.png', bbox_inches='tight') - if args.no_show: - plt.close(fig4) - - - # photon count vs. theta and phi of first hit - - fig5, ax5 = plt.subplots(figsize=(8,6)) - nbins_theta = 90 - nbins_phi = 36 - hist, x_edges, y_edges = np.histogram2d(first_hit[0], first_hit[1], - bins=[nbins_theta, nbins_phi], - range=((0, np.pi), (-np.pi, np.pi)), - weights=n_ph) - vmax=1e4 - lim=1 - if not args.no_norm: - hist = hist.astype(float)/n_evt - vmax/=n_evt - lim/=n_evt - img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], - aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) - cbar = plt.colorbar(img, ax=ax5) - cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') - - ax5.set_xlabel(r'Polar angle $\theta$ of primary particle') - ax5.set_ylabel(r'Azimuthal angle $\phi$ of primary particle') - ax5.set_title('Photon Count') - ax5.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax5.xaxis.set_major_formatter(FuncFormatter(format_func)) - ax5.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax5.yaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig5.savefig(output_start+'_phCount_thetaphi_1stHit.png', bbox_inches='tight') - if args.no_show: - plt.close(fig5) - - - else: - plt.show() - - -######################################################################### - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Process simulation" - ) - parser.add_argument('-f', "--inputFile", type=str, - help='The name of the simulation file to be processed', default='arc_pi+_50GeV.root') - parser.add_argument('-o', "--outputPath", type=str, - help='The name of the directory where to save output files', default='./') - parser.add_argument("--no_save", action='store_true', help='Do not save output arrays') - parser.add_argument("--no_show", action='store_true', help='Do not plot output histograms') - parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') - - args = parser.parse_args() - - n_ph, first_hit, ph_count = count_photons(args.inputFile) - ph_presence = (len(ph_count[0]) != 0) - - if not(args.no_save and args.no_show): - if ph_presence: - plot_ph_count(n_ph, first_hit, ph_count, args) - - print(ph_presence) - - - - - From 0bc87fdbd7875db8568e42272134f5a9363f5f31 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 5 Aug 2024 16:53:25 +0200 Subject: [PATCH 04/73] FIX typo in steering file name --- .../pipeline_scripts/ARC_standalone_o1_v01_simulation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh index b511cc6..1c03176 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh @@ -4,7 +4,7 @@ source /cvmfs/sw.hsf.org/key4hep/setup.sh echo "Sourcing executed successfully" echo "Starting simulation..." -ddsim --steeringFile CLDConfig/CLDConfig/cld_arc__steer.py \ +ddsim --steeringFile CLDConfig/CLDConfig/cld_arc_steer.py \ --enableGun --gun.distribution cos(theta) \ --gun.energy "20*GeV" --gun.particle proton \ --numberOfEvents $NUMBER_OF_EVENTS \ From 29a15228cbad7f4e2dec74eba680b098341f474c Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Tue, 6 Aug 2024 13:57:13 +0200 Subject: [PATCH 05/73] DELETE useless git clone --- .../CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh index 9f7f00d..8677a09 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh @@ -2,7 +2,6 @@ echo "Downloading necessary Github repos..." -git clone --depth 1 https://github.com/key4hep/k4geo.git git clone https://github.com/key4hep/CLDConfig.git echo "Download terminated - setup stage successful" \ No newline at end of file From b1e908d9da1897377a0ef59392f96bb21c82c9fb Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Tue, 6 Aug 2024 13:58:09 +0200 Subject: [PATCH 06/73] FIX errors when using new steering file from CLDConfig --- .../FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py | 11 +---------- .../ARC_standalone_o1_v01_simulation.sh | 10 ++++++---- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py index 923c4c1..353d973 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py @@ -13,14 +13,6 @@ import DDRec - -def find_directory(directory_name, search_path): - for root, dirs, files in os.walk(search_path): - if directory_name in dirs: - return os.path.join(root, directory_name) - return None - - def format_func(value, tick_number): N = int(np.round(value / np.pi * 6)) if N == 0: @@ -61,9 +53,8 @@ def count_photons(inputFile): podio_reader = root_io.Reader(inputFile) # get detector description for cell id decoding - k4geo_path = find_directory("k4geo", "/validation") theDetector = Detector.getInstance() - theDetector.fromXML(k4geo_path+"/test/compact/ARC_standalone_o1_v01.xml") + theDetector.fromXML(os.environ["K4GEO"]+"/FCCee/CLD/compact/CLD_o3_v01/CLD_o3_v01.xml") idposConv = DDRec.CellIDPositionConverter(theDetector) n_evts = len(podio_reader.get("events")) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh index 1c03176..fb8ca54 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh @@ -3,11 +3,13 @@ echo "Sourcing key4hep..." source /cvmfs/sw.hsf.org/key4hep/setup.sh echo "Sourcing executed successfully" +cd CLDConfig/CLDConfig echo "Starting simulation..." -ddsim --steeringFile CLDConfig/CLDConfig/cld_arc_steer.py \ - --enableGun --gun.distribution cos(theta) \ +ddsim --steeringFile cld_arc_steer.py \ + --enableGun --gun.distribution "cos(theta)" \ --gun.energy "20*GeV" --gun.particle proton \ --numberOfEvents $NUMBER_OF_EVENTS \ - --outputFile ARC_sim.root \ + --outputFile $WORKAREA/$GEOMETRY/$VERSION/ARC_sim.root \ --random.enableEventSeed --random.seed 42 -echo "Simulation ended successfully" \ No newline at end of file +echo "Simulation ended successfully" +cd $WORKAREA/$GEOMETRY/$VERSION \ No newline at end of file From 82166702b4dc9fd30a656e2a2e3182cdd295ea99 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Wed, 7 Aug 2024 19:13:18 +0200 Subject: [PATCH 07/73] ADD comparison for histos in files --- .../CLD/CLD_o3_v01/analysis/compare_histos.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py new file mode 100644 index 0000000..f19988e --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py @@ -0,0 +1,48 @@ +import ROOT +import numpy as np +import argparse +import uproot + +def compare_histos(inputFile, refFile, SL): + + if SL < 0 or SL > 1: + print('Please select a significance level between 0 and 1!') + return + + input_file = ROOT.TFile(inputFile, "READ") + ref_file = ROOT.TFile(refFile, "READ") + + matches = {} + ref_keys = ref_file.GetListOfKeys() + for key in input_file.GetListOfKeys(): + for ref_key in ref_keys: + if key.GetName() == ref_key.GetName() and \ + 'TH1' in key.GetClassName() and 'TH1' in ref_key.GetClassName(): + histo1 = input_file.Get(key.GetName()) + histo2 = ref_file.Get(key.GetName()) + p_val = histo1.Chi2Test(histo2) + + if p_val < 1-SL: + matches[key.GetName()] = False + else: + matches[key.GetName()] = True + + break + + return matches + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser( + description="Compare histograms in different files" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the file with new histos to check', default='ARC_analysis.root') + parser.add_argument('-r', "--referenceFile", type=str, + help='The name of the file containing reference histos', default='') + parser.add_argument("--SL", type=float, default=0.95, + help='Significance level of the test to perform') + args = parser.parse_args() + + compare_histos(args.inputFile, args.referenceFile, args.SL) \ No newline at end of file From 53d94e351d1a3bd0fe2568e11c450dac21583d9c Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Wed, 7 Aug 2024 19:13:50 +0200 Subject: [PATCH 08/73] ADD use ROOT files for histograms and comparison --- .../CLD/CLD_o3_v01/analysis/ARC_make_file.py | 90 ++++++ .../CLD/CLD_o3_v01/analysis/ARC_val_plots.py | 272 ++++++------------ 2 files changed, 174 insertions(+), 188 deletions(-) create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py new file mode 100644 index 0000000..fb965bf --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py @@ -0,0 +1,90 @@ +import ROOT +from podio import root_io +import dd4hep as dd4hepModule +from ROOT import dd4hep +import argparse +import numpy as np +import os +from dd4hep import Detector +import DDRec + + +def make_photon_file(args): + + # create output ROOT file + outputFile = ROOT.TFile(args.outputFile, "RECREATE") + + # set file reader + podio_reader = root_io.Reader(args.inputFile) + + # get detector description for cell id decoding + theDetector = Detector.getInstance() + theDetector.fromXML(os.environ["K4GEO"]+"/FCCee/CLD/compact/CLD_o3_v01/ARC_o1_v01.xml") + idposConv = DDRec.CellIDPositionConverter(theDetector) + + + ########## Count Photon Hits ######################### + + hist_nPh = ROOT.TH1F("h_nPh", "Total number of photon counts per event", + 50, 0, 250) + hist_theta = ROOT.TH1F("h_theta", "Number of photons counts per event vs. theta", + 90, 0, np.pi) + hist_theta_1stHit = ROOT.TH1F("h_theta_1stHit", "Number of photons counts per event vs. theta of incoming particle", + 90, 0, np.pi) + + # loop over dataset + for event in podio_reader.get("events"): + + n_ph = 0 + + p = (event.get("MCParticles"))[0] + mom = ROOT.TVector3(p.getMomentum().x, p.getMomentum().y, p.getMomentum().z) + theta_1stHit = mom.Theta() + + for arc_hit in event.get("ArcCollection"): + particle = arc_hit.getMCParticle() + pdgID = particle.getPDG() + if pdgID == 22 or pdgID == -22: + n_ph += 1 + cellID = arc_hit.getCellID() + x = idposConv.position(cellID) + theta = x.theta() + + hist_theta.Fill(theta) + + hist_nPh.Fill(n_ph) + hist_theta_1stHit.Fill(theta_1stHit, n_ph) + + hist_nPh.Write() + hist_theta.Write() + hist_theta_1stHit.Write() + + # check for photon presence + photon_presence = True + if hist_theta.GetEntries() == 0: + photon_presence = False + + outputFile.Close() + + return photon_presence + + +######################################################################### + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process simulation" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the simulation file to be processed', default='ARC_sim.root') + parser.add_argument('-o', "--outputFile", type=str, + help='The name of the ROOT file where to save output histograms', default='ARC_analysis.root') + args = parser.parse_args() + + ph_presence = make_photon_file(args) + print(ph_presence) + + + + + diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py index 353d973..d66b1f4 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py @@ -1,16 +1,34 @@ import ROOT -from podio import root_io -import dd4hep as dd4hepModule -from ROOT import dd4hep +import uproot import matplotlib.pyplot as plt from matplotlib.ticker import MultipleLocator, FuncFormatter from matplotlib.colors import SymLogNorm import argparse import numpy as np +import sys +import importlib.util +from pathlib import Path import os -from dd4hep import Detector -import DDRec + + +plot_params = { + + "nPh" : { + "range" : (0, 250), + "xlabel": 'Number of Photons', + "ylabel": 'Photon count / 5', + "xticks": False, + }, + + "theta" : { + "range" : (0, np.pi), + "xlabel": r'Polar angle $\theta$', + "ylabel": f'Photon count / {np.pi/90:.2f} rad', + "xticks": True + } + +} def format_func(value, tick_number): @@ -45,186 +63,67 @@ def format_func(value, tick_number): return r"-$\pi/6$" else: return r"${0}\pi/6$".format(N) - - -def count_photons(inputFile): - - # set file reader - podio_reader = root_io.Reader(inputFile) - - # get detector description for cell id decoding - theDetector = Detector.getInstance() - theDetector.fromXML(os.environ["K4GEO"]+"/FCCee/CLD/compact/CLD_o3_v01/CLD_o3_v01.xml") - idposConv = DDRec.CellIDPositionConverter(theDetector) - n_evts = len(podio_reader.get("events")) - - - ########## Count Photon Hits ######################### - - # prepare arrays for quantities of interest - n_ph = np.zeros(n_evts) - first_hit = np.zeros((2, n_evts)) - ph_theta = [] - ph_phi = [] - # loop over dataset - for i_evt, event in enumerate(podio_reader.get("events")): - p = (event.get("MCParticles"))[0] - mom = ROOT.TVector3(p.getMomentum().x, p.getMomentum().y, p.getMomentum().z) - first_hit[0,i_evt] = mom.Theta() - first_hit[1,i_evt] = mom.Phi() - - for i_hit, arc_hit in enumerate(event.get("ArcCollection")): - particle = arc_hit.getMCParticle() - pdgID = particle.getPDG() - if pdgID == 22 or pdgID == -22: - n_ph[i_evt] += 1 - cellID = arc_hit.getCellID() - x = idposConv.position(cellID) - ph_theta.append(x.theta()) - ph_phi.append(x.phi()) - - return n_ph, first_hit, [ph_theta, ph_phi] - - -def plot_ph_count(n_ph, first_hit, ph_count, args): - - n_evt = len(n_ph) - print("Number of evts:", n_evt) +def make_plots(args): filename = os.path.splitext(os.path.basename(args.inputFile))[0] # removes path and extension if args.outputPath[-1] != '/': args.outputPath += '/' output_start = args.outputPath + filename - - # photon count vs. theta - - fig1, ax1 = plt.subplots(figsize=(8,6)) - nbins=90 - hist, bin_edges = np.histogram(ph_count[0], bins=nbins, range=(0, np.pi)) - if not args.no_norm: - hist = hist.astype(float)/n_evt - ax1.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), - color='lightblue', edgecolor='black', align='edge') - ax1.set_xlabel(r'Polar angle $\theta$') - ax1.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') - ax1.set_yscale('log') - ax1.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax1.xaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig1.savefig(output_start+'_phCount_theta.svg', bbox_inches='tight') - if args.no_show: - plt.close(fig1) - - - # photon count vs. theta and phi - - fig2, ax2 = plt.subplots(figsize=(8,6)) - nbins_theta = 90 - nbins_phi = 36 - hist, x_edges, y_edges = np.histogram2d(ph_count[0], ph_count[1], - bins=[nbins_theta, nbins_phi], - range=((0, np.pi), (-np.pi, np.pi))) - vmax=1e4 - lim=1 - if not args.no_norm: - hist = hist.astype(float)/n_evt - vmax/=n_evt - lim/=n_evt - img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], - aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) - cbar = plt.colorbar(img, ax=ax2) - cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') - - ax2.set_xlabel(r'Polar angle $\theta$') - ax2.set_ylabel(r'Azimuthal angle $\phi$') - ax2.set_title('Photon Count') - ax2.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax2.xaxis.set_major_formatter(FuncFormatter(format_func)) - ax2.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax2.yaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig2.savefig(output_start+'_phCount_thetaphi.svg', bbox_inches='tight') - if args.no_show: - plt.close(fig2) - - - # number of photons per evt - - fig3, ax3 = plt.subplots(figsize=(8,6)) - nbins=50 - ax3.hist(n_ph, bins=nbins, range=(0, 250), - color='lightblue', edgecolor='black') - ax3.set_xlabel('Number of photons') - ax3.set_ylabel(f'Photon count / 5') - ax3.set_yscale('log') - - if not args.no_save: - fig3.savefig(output_start+'_phCount_Nph.svg', bbox_inches='tight') - if args.no_show: - plt.close(fig3) - - - # photon count vs. theta of first hit - - fig4, ax4 = plt.subplots(figsize=(8,6)) - nbins=90 - hist, bin_edges = np.histogram(first_hit[0], bins=nbins, range=(0, np.pi), weights=n_ph) - if not args.no_norm: - hist = hist.astype(float)/n_evt - ax4.bar(bin_edges[:-1], hist, width=np.diff(bin_edges), - color='lightblue', edgecolor='black', align='edge') - ax4.set_xlabel(r'Polar angle $\theta$ of primary particle') - ax4.set_ylabel(f'Photon count / {np.pi/nbins:.2f} rad') - ax4.set_yscale('log') - ax4.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax4.xaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig4.savefig(output_start+'_phCount_theta_1stHit.svg', bbox_inches='tight') - if args.no_show: - plt.close(fig4) - - - # photon count vs. theta and phi of first hit - - fig5, ax5 = plt.subplots(figsize=(8,6)) - nbins_theta = 90 - nbins_phi = 36 - hist, x_edges, y_edges = np.histogram2d(first_hit[0], first_hit[1], - bins=[nbins_theta, nbins_phi], - range=((0, np.pi), (-np.pi, np.pi)), - weights=n_ph) - vmax=1e4 - lim=1 - if not args.no_norm: - hist = hist.astype(float)/n_evt - vmax/=n_evt - lim/=n_evt - img = plt.imshow(hist.T, origin='lower', extent=[x_edges[0], x_edges[-1], y_edges[0], y_edges[-1]], - aspect='auto', cmap='viridis', norm=SymLogNorm(vmin=0, vmax=vmax, linthresh=lim)) - cbar = plt.colorbar(img, ax=ax5) - cbar.set_label(rf'Counts / {np.pi/nbins_theta:.2f} rad ($\theta$) / {2*np.pi/nbins_phi:.2f} rad ($\phi$) ') - - ax5.set_xlabel(r'Polar angle $\theta$ of primary particle') - ax5.set_ylabel(r'Azimuthal angle $\phi$ of primary particle') - ax5.set_title('Photon Count') - ax5.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax5.xaxis.set_major_formatter(FuncFormatter(format_func)) - ax5.yaxis.set_major_locator(MultipleLocator(np.pi / 6)) - ax5.yaxis.set_major_formatter(FuncFormatter(format_func)) - - if not args.no_save: - fig5.savefig(output_start+'_phCount_thetaphi_1stHit.svg', bbox_inches='tight') - if args.no_show: - plt.close(fig5) - - - else: + # read input file + inputFile = uproot.open(args.inputFile) + + # check reference + if len(args.referenceFile) != 0: + refFile = uproot.open(args.referenceFile) + compare_module = importlib.import_module("compare_histos") + histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL) + + + # Plot histograms + for key in inputFile: + histo = inputFile[key] + edges = histo.axis().edges() + values = histo.values() + + param = {} + for p in plot_params: + if p in key: + param = plot_params[p] + break + if len(param) == 0: + print(f"No plot parameters found for histogram {key}!") + continue + + fig, ax = plt.subplots(figsize=(8,6)) + ax.hist(edges[:-1], bins=edges, weights=values, histtype='step', + label='New hist', color='blue') + ax.set_xlabel(param['xlabel']) + ax.set_ylabel(param['ylabel']) + ax.set_title(f'{histo.title}') + ax.set_yscale('log') + if param['xticks']: + ax.xaxis.set_major_locator(MultipleLocator(np.pi / 6)) + ax.xaxis.set_major_formatter(FuncFormatter(format_func)) + + if len(args.referenceFile) != 0: + ref_histo = refFile[key] + ref_edges = ref_histo.axis().edges() + ref_values = ref_histo.values() + ax.hist(ref_edges[:-1], bins=ref_edges, weights=ref_values, histtype='step', + label='New hist', color='red') + ax.legend(loc='best') + if not histograms_match[key.split(';')[0]]: + fig.patch.set_facecolor('red') + + if not args.no_save: + fig.savefig(output_start+f'_{histo.name}'+'.svg', bbox_inches='tight') + if not args.show: + plt.close(fig) + + if args.show: plt.show() @@ -235,23 +134,20 @@ def plot_ph_count(n_ph, first_hit, ph_count, args): description="Process simulation" ) parser.add_argument('-f', "--inputFile", type=str, - help='The name of the simulation file to be processed', default='arc_pi+_50GeV.root') + help='The name of the simulation file to be processed', default='ARC_analysis.root') parser.add_argument('-o', "--outputPath", type=str, help='The name of the directory where to save output files', default='./') + parser.add_argument('-r', "--referenceFile", type=str, + help='The name of the file containing reference histos', default='') parser.add_argument("--no_save", action='store_true', help='Do not save output arrays') - parser.add_argument("--no_show", action='store_true', help='Do not plot output histograms') - parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') + parser.add_argument("--SL", type=float, default=0.95, + help='Significance level of the test to perform') + parser.add_argument("--show", action='store_true', help='Plot output histograms') + #parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') args = parser.parse_args() - n_ph, first_hit, ph_count = count_photons(args.inputFile) - ph_presence = (len(ph_count[0]) != 0) - - if not(args.no_save and args.no_show): - if ph_presence: - plot_ph_count(n_ph, first_hit, ph_count, args) - - print(ph_presence) + make_plots(args) From 02f8ee2e6d88c93ef9c8c4ec5314fe7e544ba00e Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Thu, 8 Aug 2024 17:28:59 +0200 Subject: [PATCH 09/73] FIX k4geo repo --- .../CLD/CLD_o3_v01/analysis/ARC_make_file.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py index fb965bf..7d3f973 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py @@ -8,6 +8,32 @@ from dd4hep import Detector import DDRec +def find_directory_upwards(start_dir, target_dir_name): + """ + Recursively searches upwards from the start directory for a directory with the specified name. + + :param start_dir: The starting directory path. + :param target_dir_name: The name of the directory to search for. + :return: The path to the found directory or None if not found. + """ + current_dir = os.path.abspath(start_dir) + + while True: + # Check if the target directory exists in the current directory + if target_dir_name in os.listdir(current_dir): + possible_match = os.path.join(current_dir, target_dir_name) + if os.path.isdir(possible_match): + return possible_match + + # Move up one level in the directory tree + parent_dir = os.path.dirname(current_dir) + + # If we have reached the root directory and haven't found the directory, return None + if current_dir == parent_dir: + return None + + current_dir = parent_dir + def make_photon_file(args): @@ -18,8 +44,9 @@ def make_photon_file(args): podio_reader = root_io.Reader(args.inputFile) # get detector description for cell id decoding + k4geo = find_directory_upwards('./', 'k4geo') theDetector = Detector.getInstance() - theDetector.fromXML(os.environ["K4GEO"]+"/FCCee/CLD/compact/CLD_o3_v01/ARC_o1_v01.xml") + theDetector.fromXML(k4geo+"/test/compact/ARC_standalone_o1_v01.xml") idposConv = DDRec.CellIDPositionConverter(theDetector) @@ -55,6 +82,12 @@ def make_photon_file(args): hist_nPh.Fill(n_ph) hist_theta_1stHit.Fill(theta_1stHit, n_ph) + if not args.no_norm: + factor = 1./hist_nPh.GetEntries() + hist_nPh.Scale(factor) + hist_theta.Scale(factor) + hist_theta_1stHit.Scale(factor) + hist_nPh.Write() hist_theta.Write() hist_theta_1stHit.Write() @@ -79,6 +112,8 @@ def make_photon_file(args): help='The name of the simulation file to be processed', default='ARC_sim.root') parser.add_argument('-o', "--outputFile", type=str, help='The name of the ROOT file where to save output histograms', default='ARC_analysis.root') + parser.add_argument('--no_norm', action='store_true', + help='Do not normalize output histograms by number of events') args = parser.parse_args() ph_presence = make_photon_file(args) From e7568a2d5699a0e87b92f7d5193db93817dfd13c Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Thu, 8 Aug 2024 17:29:53 +0200 Subject: [PATCH 10/73] FIX k4geo repo --- .../pipeline_scripts/ARC_standalone_o1_v01_simulation.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh index fb8ca54..638b46e 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh @@ -10,6 +10,7 @@ ddsim --steeringFile cld_arc_steer.py \ --gun.energy "20*GeV" --gun.particle proton \ --numberOfEvents $NUMBER_OF_EVENTS \ --outputFile $WORKAREA/$GEOMETRY/$VERSION/ARC_sim.root \ - --random.enableEventSeed --random.seed 42 + --random.enableEventSeed --random.seed 42 \ + --part.userParticleHandler='' echo "Simulation ended successfully" cd $WORKAREA/$GEOMETRY/$VERSION \ No newline at end of file From 9a20f6bdb9c02d07a1ca2592389284803f044e77 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Thu, 8 Aug 2024 17:44:03 +0200 Subject: [PATCH 11/73] ADD single script for all stages --- .../ARC_standalone_o1_v01_script.sh | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh new file mode 100644 index 0000000..f56b676 --- /dev/null +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -0,0 +1,60 @@ +# setup phase +echo "SETUP PHASE:" + +echo "Sourcing key4hep..." +source /cvmfs/sw.hsf.org/key4hep/setup.sh +echo "Sourcing executed successfully" + +echo "Downloading necessary Github repos..." +git clone https://github.com/key4hep/CLDConfig.git +git clone https://github.com/key4hep/k4geo.git +echo "Download terminated - setup stage successful" + + +# simulation phase +echo "SIMULATION PHASE" + +cd CLDConfig/CLDConfig +echo "Starting simulation..." +ddsim --steeringFile cld_arc_steer.py \ + --enableGun --gun.distribution "cos(theta)" \ + --gun.energy "20*GeV" --gun.particle proton \ + --numberOfEvents $NUMBER_OF_EVENTS \ + --outputFile $WORKAREA/$GEOMETRY/$VERSION/ARC_sim.root \ + --random.enableEventSeed --random.seed 42 \ + --part.userParticleHandler='' +echo "Simulation ended successfully" +cd $WORKAREA/$GEOMETRY/$VERSION + + +# analyze simulation file +echo "ANALYSIS PHASE" + +echo "Starting analysis script..." +python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_make_file.py \ + -f ARC_sim.root -o ARC_res.root +echo "Script executed successfully" + + +# check if reference is needed +if [ "$MAKE_REFERENCE_SAMPLE" == "yes" ]; then + mv ARC_res.root $REFERENCE/ref_$VERSION.root +else + # make plots + echo "PLOT PHASE" + + echo "Starting plotting script..." + python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ + -f ARC_res.root -r $REFERENCE/ref_$VERSION.root \ + -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots + echo "Script executed successfully" + + # upload them on website + echo "Starting website script..." + cd key4hep-reco-validation/web/python + python3 make_web.py --dest $WORKAREA/$PLOTAREA + echo "Script executed successfully" +fi + + + From 7869ad4daa6cfe7799316a08065a3c9de0bc42af Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 14:38:24 +0200 Subject: [PATCH 12/73] FIX typo with reference directory name --- .../pipeline_scripts/ARC_standalone_o1_v01_script.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh index f56b676..3bdf5ff 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -38,14 +38,15 @@ echo "Script executed successfully" # check if reference is needed if [ "$MAKE_REFERENCE_SAMPLE" == "yes" ]; then - mv ARC_res.root $REFERENCE/ref_$VERSION.root + mkdir -p $WORKAREA/$GEOMETRY/$VERSION/$REFERENCE_SAMPLE + mv ARC_res.root $WORKAREA/$GEOMETRY/$VERSION/$REFERENCE_SAMPLE/ref_$VERSION.root else # make plots echo "PLOT PHASE" echo "Starting plotting script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ - -f ARC_res.root -r $REFERENCE/ref_$VERSION.root \ + -f ARC_res.root -r $REFERENCE_SAMPLE/ref_$VERSION.root \ -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots echo "Script executed successfully" From b86882551aff8919348d5b97b869e8778169c3e7 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 15:52:02 +0200 Subject: [PATCH 13/73] FIX compact file for simulation --- .../CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh index 3bdf5ff..6b7f0fe 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -17,6 +17,7 @@ echo "SIMULATION PHASE" cd CLDConfig/CLDConfig echo "Starting simulation..." ddsim --steeringFile cld_arc_steer.py \ + --compactFile ../../k4geo/test/compact/ARC_standalone_o1_v01.xml \ --enableGun --gun.distribution "cos(theta)" \ --gun.energy "20*GeV" --gun.particle proton \ --numberOfEvents $NUMBER_OF_EVENTS \ From 1092189c3279f15855b9b612fc434af9ad247df7 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 16:06:44 +0200 Subject: [PATCH 14/73] FIX reference file directory --- .../pipeline_scripts/ARC_standalone_o1_v01_script.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh index 6b7f0fe..38d60b7 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -39,15 +39,15 @@ echo "Script executed successfully" # check if reference is needed if [ "$MAKE_REFERENCE_SAMPLE" == "yes" ]; then - mkdir -p $WORKAREA/$GEOMETRY/$VERSION/$REFERENCE_SAMPLE - mv ARC_res.root $WORKAREA/$GEOMETRY/$VERSION/$REFERENCE_SAMPLE/ref_$VERSION.root + mkdir -p $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION + mv ARC_res.root $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root else # make plots echo "PLOT PHASE" echo "Starting plotting script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ - -f ARC_res.root -r $REFERENCE_SAMPLE/ref_$VERSION.root \ + -f ARC_res.root -r $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root \ -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots echo "Script executed successfully" From 3b195b387bcbd666fab50b3e3a7ab9d9b3f9fae3 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 16:08:00 +0200 Subject: [PATCH 15/73] ADD redirect simulation output to log file --- .../CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh index 38d60b7..a8c7486 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -23,7 +23,7 @@ ddsim --steeringFile cld_arc_steer.py \ --numberOfEvents $NUMBER_OF_EVENTS \ --outputFile $WORKAREA/$GEOMETRY/$VERSION/ARC_sim.root \ --random.enableEventSeed --random.seed 42 \ - --part.userParticleHandler='' + --part.userParticleHandler='' > $WORKAREA/$GEOMETRY/$VERSION/ddsim_log.txt echo "Simulation ended successfully" cd $WORKAREA/$GEOMETRY/$VERSION From 6bdff566139dc838b732c3c30fe6026b67548ab3 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 17:02:16 +0200 Subject: [PATCH 16/73] ADD verbosity --- scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py index f19988e..7a5b75b 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py @@ -27,6 +27,8 @@ def compare_histos(inputFile, refFile, SL): else: matches[key.GetName()] = True + print(f"P-value for histo {key.GetName()}: {p_val} --> match test {'passed' if matches[key.GetName()] else 'failed'}") + break return matches From 063d3c6f93f68da72bde80a1fba2512b8c3fc752 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 17:02:40 +0200 Subject: [PATCH 17/73] ADD absence of reference histogram case --- .../CLD/CLD_o3_v01/analysis/ARC_val_plots.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py index d66b1f4..765751c 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py @@ -109,14 +109,26 @@ def make_plots(args): ax.xaxis.set_major_formatter(FuncFormatter(format_func)) if len(args.referenceFile) != 0: - ref_histo = refFile[key] - ref_edges = ref_histo.axis().edges() - ref_values = ref_histo.values() - ax.hist(ref_edges[:-1], bins=ref_edges, weights=ref_values, histtype='step', - label='New hist', color='red') - ax.legend(loc='best') - if not histograms_match[key.split(';')[0]]: - fig.patch.set_facecolor('red') + try: + ref_histo = refFile[key] + ref_edges = ref_histo.axis().edges() + ref_values = ref_histo.values() + ax.hist(ref_edges[:-1], bins=ref_edges, weights=ref_values, histtype='step', + label='Reference hist', color='red') + ax.legend(loc='best') + if not histograms_match[key.split(';')[0]]: + fig.patch.set_facecolor('red') + except uproot.exceptions.KeyInFileError: + fig.patch.set_facecolor('yellow') + plt.text( + 0.95, 0.05, + "WARNING: Reference histogram not found", + fontsize=12, + color='black', + ha='right', # Horizontal alignment: right + va='bottom', # Vertical alignment: bottom + transform=plt.gca().transAxes # Use the Axes coordinates (relative coordinates) + ) if not args.no_save: fig.savefig(output_start+f'_{histo.name}'+'.svg', bbox_inches='tight') From ef1e67a36c24d1b83a247994bfa956081c8ba4a6 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Fri, 9 Aug 2024 17:03:07 +0200 Subject: [PATCH 18/73] FIX histogram names --- scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py index 7d3f973..b748bdc 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py @@ -54,9 +54,9 @@ def make_photon_file(args): hist_nPh = ROOT.TH1F("h_nPh", "Total number of photon counts per event", 50, 0, 250) - hist_theta = ROOT.TH1F("h_theta", "Number of photons counts per event vs. theta", + hist_theta = ROOT.TH1F("h_theta", "Average number of photons counts per event vs. theta", 90, 0, np.pi) - hist_theta_1stHit = ROOT.TH1F("h_theta_1stHit", "Number of photons counts per event vs. theta of incoming particle", + hist_theta_1stHit = ROOT.TH1F("h_theta_1stHit", "Average number of photons counts per event vs. theta of incoming particle", 90, 0, np.pi) # loop over dataset @@ -84,7 +84,6 @@ def make_photon_file(args): if not args.no_norm: factor = 1./hist_nPh.GetEntries() - hist_nPh.Scale(factor) hist_theta.Scale(factor) hist_theta_1stHit.Scale(factor) From 8ace834de99608981923c42d0d7504223c5f04ab Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 14:12:08 +0200 Subject: [PATCH 19/73] ADD kolmogorov and identical tests --- .../CLD/CLD_o3_v01/analysis/ARC_val_plots.py | 3 +- .../CLD/CLD_o3_v01/analysis/compare_histos.py | 70 ++++++++++++++++--- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py index 765751c..b8b913c 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py @@ -79,7 +79,7 @@ def make_plots(args): if len(args.referenceFile) != 0: refFile = uproot.open(args.referenceFile) compare_module = importlib.import_module("compare_histos") - histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL) + histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) # Plot histograms @@ -155,6 +155,7 @@ def make_plots(args): parser.add_argument("--SL", type=float, default=0.95, help='Significance level of the test to perform') parser.add_argument("--show", action='store_true', help='Plot output histograms') + parser.add_argument("--test", type=str, help=f"Test to check compatibility of histograms. Possible options are: chi2, KS, identical") #parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') args = parser.parse_args() diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py index 7a5b75b..ae86408 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py @@ -3,11 +3,65 @@ import argparse import uproot -def compare_histos(inputFile, refFile, SL): + +def Chi2test(h1, h2, SL): + p_val = h1.Chi2Test(h2) + if p_val < 1-SL: + match = False + else: + match = True + print(f"P-value for histo {h1.GetName()}: {p_val} --> match test {'passed' if match else 'failed'}") + + return match + +def KStest(h1, h2, SL): + p_val = h1.KolmogorovTest(h2) + if p_val < 1-SL: + match = False + else: + match = True + print(f"P-value for histo {h1.GetName()}: {p_val} --> match test {'passed' if match else 'failed'}") + + return match + +def Identical(h1, h2, SL): + if h1.GetNbinsX() != h2.GetNbinsX(): + print("Number of bins does not match.") + return False + + for bin_num in range(1, h1.GetNbinsX() + 1): + bin_edge1 = h1.GetBinLowEdge(bin_num) + bin_edge2 = h2.GetBinLowEdge(bin_num) + + bin_content1 = h1.GetBinContent(bin_num) + bin_content2 = h2.GetBinContent(bin_num) + + if bin_edge1 != bin_edge2: + print(f"Bin edges do not match at bin {bin_num}: {bin_edge1} vs {bin_edge2}") + return False + + if bin_content1 != bin_content2: + print(f"Bin contents do not match at bin {bin_num}: {bin_content1} vs {bin_content2}") + return False + + print(f"Histograms {h1.GetName()} match!") + return True + + +tests = { + "chi2" : Chi2test, + "KS" : KStest, + "identical" : Identical +} + + +def compare_histos(inputFile, refFile, SL, test_name): if SL < 0 or SL > 1: print('Please select a significance level between 0 and 1!') return + + test = tests[test_name] input_file = ROOT.TFile(inputFile, "READ") ref_file = ROOT.TFile(refFile, "READ") @@ -20,14 +74,8 @@ def compare_histos(inputFile, refFile, SL): 'TH1' in key.GetClassName() and 'TH1' in ref_key.GetClassName(): histo1 = input_file.Get(key.GetName()) histo2 = ref_file.Get(key.GetName()) - p_val = histo1.Chi2Test(histo2) - - if p_val < 1-SL: - matches[key.GetName()] = False - else: - matches[key.GetName()] = True - - print(f"P-value for histo {key.GetName()}: {p_val} --> match test {'passed' if matches[key.GetName()] else 'failed'}") + + matches[key.GetName()] = test(histo1, histo2, SL) break @@ -45,6 +93,8 @@ def compare_histos(inputFile, refFile, SL): help='The name of the file containing reference histos', default='') parser.add_argument("--SL", type=float, default=0.95, help='Significance level of the test to perform') + dict_keys = ', '.join(tests.keys()) + parser.add_argument("--test", type=str, choices=tests.keys(), help=f"Test to check compatibility of histograms. Possible options are: {dict_keys}") args = parser.parse_args() - compare_histos(args.inputFile, args.referenceFile, args.SL) \ No newline at end of file + compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) \ No newline at end of file From b759ff06d3cd55644d0f3da9adc9c68441ce10fd Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 14:13:26 +0200 Subject: [PATCH 20/73] check if histograms are perfectly identical' --- .../CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh index a8c7486..6077cea 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh @@ -48,7 +48,7 @@ else echo "Starting plotting script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ -f ARC_res.root -r $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root \ - -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots + -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots --test identical echo "Script executed successfully" # upload them on website From f86566cd741610a93860610dda6192aaf5a230d9 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 16:13:01 +0200 Subject: [PATCH 21/73] ADD ALLEGRO test --- .../ALLEGRO_o1_v03/ALLEGRO_make_file.py | 78 +++++++++++ .../ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh | 48 +++++++ .../ALLEGRO_o1_v03/ALLEGRO_val_plots.py | 129 ++++++++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py create mode 100644 scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh create mode 100644 scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py new file mode 100644 index 0000000..66e4782 --- /dev/null +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py @@ -0,0 +1,78 @@ +import ROOT +from podio import root_io +import dd4hep as dd4hepModule +from ROOT import dd4hep +import argparse +import numpy as np +import os +from dd4hep import Detector +import DDRec + + +def make_file(args): + + # create output ROOT file + outputFile = ROOT.TFile(args.outputFile, "RECREATE") + + # set file reader + podio_reader = root_io.Reader(args.inputFile) + + # get detector description for cell id decoding + #theDetector = Detector.getInstance() + #theDetector.fromXML(os.environ["$K4GEO"]+"/FCCee/ALLEGRO/compact/ALLEGRO_o1_v03/ALLEGRO_o1_v03.xml") + #idposConv = DDRec.CellIDPositionConverter(theDetector) + + + ########## Count Photon Hits ######################### + + hist_ccE = ROOT.TH1F("h_CaloCluster_E", "CaloCluster Energy", + 0, 15, 100) + hist_ctcE = ROOT.TH1F("h_CaloTopoCluster_E", "CaloTopoCluster Energy", + 0, 10, 100) + + # loop over dataset + for event in podio_reader.get("events"): + + for calo in event.get("CaloClusters"): + energy = calo.getEnergy() + hist_ccE.Fill(energy) + + for calo in event.get("CaloTopoClusters"): + energy = calo.energy() + hist_ctcE.Fill(energy) + + n_evts = len(podio_reader.get("events")) + + if not args.no_norm: + factor = 1./n_evts + hist_ccE.Scale(factor) + hist_ctcE.Scale(factor) + + hist_ccE.Write() + hist_ctcE.Write() + + outputFile.Close() + + return + + +######################################################################### + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process simulation" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the simulation file to be processed', default='ARC_sim.root') + parser.add_argument('-o', "--outputFile", type=str, + help='The name of the ROOT file where to save output histograms', default='ARC_analysis.root') + parser.add_argument('--no_norm', action='store_true', + help='Do not normalize output histograms by number of events') + args = parser.parse_args() + + make_file(args) + + + + + diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh new file mode 100644 index 0000000..efc8ae9 --- /dev/null +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh @@ -0,0 +1,48 @@ +printf "\n\nSETUP PHASE:\n" + +echo "Downloading necessary Github repos..." +git clone https://github.com/HEP-FCC/FCC-config.git +echo "Download terminated - setup stage successful" + + +# simulation phase +printf "\n\nSIM-DIGI-RECO PHASE:\n" + +cd FCC-config/FCCee/FullSim/ALLEGRO/ALLEGRO_o1_v03 +echo "Starting script..." +./ctest_sim_digi_reco.sh +mv ALLEGRO_sim_digi_reco.root $WORKAREA/$GEOMETRY/$VERSION/ + + +# analyze simulation file +printf "\n\nANALYSIS PHASE:\n" + +echo "Starting analysis script..." +python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/ALLEGRO_make_file.py \ + -f ALLEGRO_sim_digi_reco.root -o ALLEGRO_res.root +echo "Script executed successfully" + + +# check if reference is needed +if [ "$MAKE_REFERENCE_SAMPLE" == "yes" ]; then + mkdir -p $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION + mv ALLEGRO_res.root $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root +else + # make plots + printf "\n\nPLOT PHASE:\n" + + echo "Starting plotting script..." + python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/ALLEGRO_val_plots.py \ + -f ALLEGRO_res.root -r $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root \ + -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots --test identical + echo "Script executed successfully" + + # upload them on website + echo "Starting website script..." + cd key4hep-reco-validation/web/python + python3 make_web.py --dest $WORKAREA/$PLOTAREA + echo "Script executed successfully" +fi + + + diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py new file mode 100644 index 0000000..e06627b --- /dev/null +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py @@ -0,0 +1,129 @@ +import ROOT +import uproot +import matplotlib.pyplot as plt +from matplotlib.ticker import MultipleLocator, FuncFormatter +from matplotlib.colors import SymLogNorm + +import argparse +import numpy as np +import sys +import importlib.util +from pathlib import Path +import os + + +plot_params = { + + "ccE" : { + "range" : (0, 15), + "xlabel": 'Energy [MeV]', + "ylabel": 'Counts / 0.15 MeV', + }, + + "ctcE" : { + "range" : (0, 10), + "xlabel": 'Energy [MeV]', + "ylabel": 'Counts / 0.1 MeV', + } + +} + + +def make_plots(args): + + filename = os.path.splitext(os.path.basename(args.inputFile))[0] # removes path and extension + if args.outputPath[-1] != '/': + args.outputPath += '/' + output_start = args.outputPath + filename + + # read input file + inputFile = uproot.open(args.inputFile) + + # check reference + if len(args.referenceFile) != 0: + refFile = uproot.open(args.referenceFile) + compare_module = importlib.import_module("compare_histos") + histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) + + + # Plot histograms + for key in inputFile: + histo = inputFile[key] + edges = histo.axis().edges() + values = histo.values() + + param = {} + for p in plot_params: + if p in key: + param = plot_params[p] + break + if len(param) == 0: + print(f"No plot parameters found for histogram {key}!") + continue + + fig, ax = plt.subplots(figsize=(8,6)) + ax.hist(edges[:-1], bins=edges, weights=values, histtype='step', + label='New hist', color='blue') + ax.set_xlabel(param['xlabel']) + ax.set_ylabel(param['ylabel']) + ax.set_title(f'{histo.title}') + ax.set_yscale('log') + + if len(args.referenceFile) != 0: + try: + ref_histo = refFile[key] + ref_edges = ref_histo.axis().edges() + ref_values = ref_histo.values() + ax.hist(ref_edges[:-1], bins=ref_edges, weights=ref_values, histtype='step', + label='Reference hist', color='red') + ax.legend(loc='best') + if not histograms_match[key.split(';')[0]]: + fig.patch.set_facecolor('red') + except uproot.exceptions.KeyInFileError: + fig.patch.set_facecolor('yellow') + plt.text( + 0.95, 0.05, + "WARNING: Reference histogram not found", + fontsize=12, + color='black', + ha='right', # Horizontal alignment: right + va='bottom', # Vertical alignment: bottom + transform=plt.gca().transAxes # Use the Axes coordinates (relative coordinates) + ) + + if not args.no_save: + fig.savefig(output_start+f'_{histo.name}'+'.svg', bbox_inches='tight') + if not args.show: + plt.close(fig) + + if args.show: + plt.show() + + +######################################################################### + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process simulation" + ) + parser.add_argument('-f', "--inputFile", type=str, + help='The name of the simulation file to be processed', default='ARC_analysis.root') + parser.add_argument('-o', "--outputPath", type=str, + help='The name of the directory where to save output files', default='./') + parser.add_argument('-r', "--referenceFile", type=str, + help='The name of the file containing reference histos', default='') + parser.add_argument("--no_save", action='store_true', help='Do not save output arrays') + parser.add_argument("--SL", type=float, default=0.95, + help='Significance level of the test to perform') + parser.add_argument("--show", action='store_true', help='Plot output histograms') + parser.add_argument("--test", type=str, help=f"Test to check compatibility of histograms. Possible options are: chi2, KS, identical") + #parser.add_argument("--no_norm", action='store_true', help='Do not normalize output histograms by number of events') + + args = parser.parse_args() + + make_plots(args) + + + + + From 6940fb7077c6510999c0f5e917e2d13887469f8f Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 16:44:06 +0200 Subject: [PATCH 22/73] Change repository structure --- .../ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py | 13 +++++++++++-- .../CLD_o3_v01/{analysis => }/ARC_make_file.py | 0 .../ARC_standalone_o1_v01_script.sh | 0 .../CLD_o3_v01/{analysis => }/ARC_val_plots.py | 13 +++++++++++-- .../ARC_standalone_o1_v01_plot.sh | 16 ---------------- .../ARC_standalone_o1_v01_setup.sh | 7 ------- .../ARC_standalone_o1_v01_simulation.sh | 16 ---------------- .../CLD_o3_v01/analysis => }/compare_histos.py | 0 8 files changed, 22 insertions(+), 43 deletions(-) rename scripts/FCCee/CLD/CLD_o3_v01/{analysis => }/ARC_make_file.py (100%) rename scripts/FCCee/CLD/CLD_o3_v01/{pipeline_scripts => }/ARC_standalone_o1_v01_script.sh (100%) rename scripts/FCCee/CLD/CLD_o3_v01/{analysis => }/ARC_val_plots.py (90%) delete mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh delete mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh delete mode 100644 scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh rename scripts/{FCCee/CLD/CLD_o3_v01/analysis => }/compare_histos.py (100%) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py index e06627b..46051b1 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py @@ -41,10 +41,19 @@ def make_plots(args): # check reference if len(args.referenceFile) != 0: + # import compare_histos module from scripts directory + current_dir = Path(__file__).resolve().parent + target_dir = current_dir.parents[3] + sys.path.insert(0, str(target_dir)) + module_name = "compare_histos" + spec = importlib.util.find_spec(module_name) + target_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(target_module) + refFile = uproot.open(args.referenceFile) - compare_module = importlib.import_module("compare_histos") - histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) + histograms_match = target_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) + sys.path.pop(0) # Plot histograms for key in inputFile: diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py b/scripts/FCCee/CLD/CLD_o3_v01/ARC_make_file.py similarity index 100% rename from scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_make_file.py rename to scripts/FCCee/CLD/CLD_o3_v01/ARC_make_file.py diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh similarity index 100% rename from scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_script.sh rename to scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py similarity index 90% rename from scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py rename to scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py index b8b913c..9b74536 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/analysis/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py @@ -77,10 +77,19 @@ def make_plots(args): # check reference if len(args.referenceFile) != 0: + # import compare_histos module from scripts directory + current_dir = Path(__file__).resolve().parent + target_dir = current_dir.parents[3] + sys.path.insert(0, str(target_dir)) + module_name = "compare_histos" + spec = importlib.util.find_spec(module_name) + target_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(target_module) + refFile = uproot.open(args.referenceFile) - compare_module = importlib.import_module("compare_histos") - histograms_match = compare_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) + histograms_match = target_module.compare_histos(args.inputFile, args.referenceFile, args.SL, args.test) + sys.path.pop(0) # Plot histograms for key in inputFile: diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh deleted file mode 100644 index e295fd1..0000000 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_plot.sh +++ /dev/null @@ -1,16 +0,0 @@ -# make plots -echo "Sourcing key4hep..." -source /cvmfs/sw.hsf.org/key4hep/setup.sh -echo "Sourcing executed successfully" - -echo "Starting analysis and plot script..." -python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ - -f ARC_sim.root --no_show \ - -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots -echo "Script executed successfully" - -# upload them on website -echo "Starting website script..." -cd key4hep-reco-validation/web/python -python3 make_web.py --dest $WORKAREA/$PLOTAREA -echo "Script executed successfully" \ No newline at end of file diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh deleted file mode 100644 index 8677a09..0000000 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_setup.sh +++ /dev/null @@ -1,7 +0,0 @@ -# setup phase - -echo "Downloading necessary Github repos..." - -git clone https://github.com/key4hep/CLDConfig.git - -echo "Download terminated - setup stage successful" \ No newline at end of file diff --git a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh b/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh deleted file mode 100644 index 638b46e..0000000 --- a/scripts/FCCee/CLD/CLD_o3_v01/pipeline_scripts/ARC_standalone_o1_v01_simulation.sh +++ /dev/null @@ -1,16 +0,0 @@ -# simulation phase -echo "Sourcing key4hep..." -source /cvmfs/sw.hsf.org/key4hep/setup.sh -echo "Sourcing executed successfully" - -cd CLDConfig/CLDConfig -echo "Starting simulation..." -ddsim --steeringFile cld_arc_steer.py \ - --enableGun --gun.distribution "cos(theta)" \ - --gun.energy "20*GeV" --gun.particle proton \ - --numberOfEvents $NUMBER_OF_EVENTS \ - --outputFile $WORKAREA/$GEOMETRY/$VERSION/ARC_sim.root \ - --random.enableEventSeed --random.seed 42 \ - --part.userParticleHandler='' -echo "Simulation ended successfully" -cd $WORKAREA/$GEOMETRY/$VERSION \ No newline at end of file diff --git a/scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py b/scripts/compare_histos.py similarity index 100% rename from scripts/FCCee/CLD/CLD_o3_v01/analysis/compare_histos.py rename to scripts/compare_histos.py From 7e0149e33381948548f2c8f642f2ba2d18f91f2e Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 16:44:41 +0200 Subject: [PATCH 23/73] add clarity to print output --- .../CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh index 6077cea..1c8006b 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh @@ -1,8 +1,8 @@ # setup phase -echo "SETUP PHASE:" +printf "\n\nSETUP PHASE:\n" echo "Sourcing key4hep..." -source /cvmfs/sw.hsf.org/key4hep/setup.sh +source /cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh echo "Sourcing executed successfully" echo "Downloading necessary Github repos..." @@ -12,7 +12,7 @@ echo "Download terminated - setup stage successful" # simulation phase -echo "SIMULATION PHASE" +printf "\n\nSIMULATION PHASE:\n" cd CLDConfig/CLDConfig echo "Starting simulation..." @@ -29,7 +29,7 @@ cd $WORKAREA/$GEOMETRY/$VERSION # analyze simulation file -echo "ANALYSIS PHASE" +printf "\n\nANALYSIS PHASE:\n" echo "Starting analysis script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_make_file.py \ @@ -43,7 +43,7 @@ if [ "$MAKE_REFERENCE_SAMPLE" == "yes" ]; then mv ARC_res.root $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root else # make plots - echo "PLOT PHASE" + printf "\n\nPLOT PHASE:\n" echo "Starting plotting script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ From d8ef39501b18e5695cceb65349ccbd0de64b15b1 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 16:52:34 +0200 Subject: [PATCH 24/73] change dirctory structure --- scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh b/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh index 1c8006b..109b7f6 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh +++ b/scripts/FCCee/CLD/CLD_o3_v01/ARC_standalone_o1_v01_script.sh @@ -32,7 +32,7 @@ cd $WORKAREA/$GEOMETRY/$VERSION printf "\n\nANALYSIS PHASE:\n" echo "Starting analysis script..." -python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_make_file.py \ +python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/ARC_make_file.py \ -f ARC_sim.root -o ARC_res.root echo "Script executed successfully" @@ -46,7 +46,7 @@ else printf "\n\nPLOT PHASE:\n" echo "Starting plotting script..." - python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/analysis/ARC_val_plots.py \ + python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/ARC_val_plots.py \ -f ARC_res.root -r $WORKAREA/$REFERENCE_SAMPLE/$GEOMETRY/$VERSION/ref_$VERSION.root \ -o $WORKAREA/$PLOTAREA/$GEOMETRY/$VERSION/plots --test identical echo "Script executed successfully" From 97fd774bb23268b73ea22c1e31e0b81b0f64cae6 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 16:52:54 +0200 Subject: [PATCH 25/73] FIX working directory --- scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh index efc8ae9..a37c443 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh @@ -12,7 +12,7 @@ cd FCC-config/FCCee/FullSim/ALLEGRO/ALLEGRO_o1_v03 echo "Starting script..." ./ctest_sim_digi_reco.sh mv ALLEGRO_sim_digi_reco.root $WORKAREA/$GEOMETRY/$VERSION/ - +cd $WORKAREA/$GEOMETRY/$VERSION # analyze simulation file printf "\n\nANALYSIS PHASE:\n" From 74a60d6d94a83d98186e71f055de3864cb14a173 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 17:12:36 +0200 Subject: [PATCH 26/73] FIX histogram initialization --- scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py index 66e4782..3df863a 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py @@ -26,9 +26,9 @@ def make_file(args): ########## Count Photon Hits ######################### hist_ccE = ROOT.TH1F("h_CaloCluster_E", "CaloCluster Energy", - 0, 15, 100) + 100, 0, 15) hist_ctcE = ROOT.TH1F("h_CaloTopoCluster_E", "CaloTopoCluster Energy", - 0, 10, 100) + 100, 0, 10) # loop over dataset for event in podio_reader.get("events"): From f76af95c031c6e27f962375bcf87fce7e48c0fcc Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 17:13:00 +0200 Subject: [PATCH 27/73] FIX sourcing nightlies --- scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh index a37c443..e0d4cb3 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_o1_v03_script.sh @@ -17,6 +17,7 @@ cd $WORKAREA/$GEOMETRY/$VERSION # analyze simulation file printf "\n\nANALYSIS PHASE:\n" +source /cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh echo "Starting analysis script..." python key4hep-reco-validation/scripts/FCCee/$GEOMETRY/$VERSION/ALLEGRO_make_file.py \ -f ALLEGRO_sim_digi_reco.root -o ALLEGRO_res.root From 09f4aff34119aa20481ce798e7a1f0fef7849ff4 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 17:40:06 +0200 Subject: [PATCH 28/73] FIX path for compare_histos --- scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py | 5 ++--- scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py index 46051b1..fd8d1f0 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py @@ -41,9 +41,8 @@ def make_plots(args): # check reference if len(args.referenceFile) != 0: - # import compare_histos module from scripts directory - current_dir = Path(__file__).resolve().parent - target_dir = current_dir.parents[3] + # import compare_histos module from scripts directory + target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../')) sys.path.insert(0, str(target_dir)) module_name = "compare_histos" spec = importlib.util.find_spec(module_name) diff --git a/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py b/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py index 9b74536..f7cb390 100644 --- a/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py +++ b/scripts/FCCee/CLD/CLD_o3_v01/ARC_val_plots.py @@ -78,8 +78,7 @@ def make_plots(args): # check reference if len(args.referenceFile) != 0: # import compare_histos module from scripts directory - current_dir = Path(__file__).resolve().parent - target_dir = current_dir.parents[3] + target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../')) sys.path.insert(0, str(target_dir)) module_name = "compare_histos" spec = importlib.util.find_spec(module_name) From f3fcc2b98d0ccd330b18d1281e6b73e86a8d10a3 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Mon, 12 Aug 2024 17:47:07 +0200 Subject: [PATCH 29/73] FIX plot params keys --- scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py index fd8d1f0..ad1337e 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py @@ -14,13 +14,13 @@ plot_params = { - "ccE" : { + "CaloCluster" : { "range" : (0, 15), "xlabel": 'Energy [MeV]', "ylabel": 'Counts / 0.15 MeV', }, - "ctcE" : { + "CaloTopoCluster" : { "range" : (0, 10), "xlabel": 'Energy [MeV]', "ylabel": 'Counts / 0.1 MeV', From 7d4400062e4c320e20af7064ac93bc0f2a1c34bf Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Tue, 13 Aug 2024 12:01:49 +0200 Subject: [PATCH 30/73] ADD info for ECalBarrelModuleThetaMergedPosition --- .../ALLEGRO_o1_v03/ALLEGRO_make_file.py | 31 +++++++++++++++++-- .../ALLEGRO_o1_v03/ALLEGRO_val_plots.py | 31 ++++++++++++++----- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py index 3df863a..5ffbc87 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_make_file.py @@ -28,7 +28,20 @@ def make_file(args): hist_ccE = ROOT.TH1F("h_CaloCluster_E", "CaloCluster Energy", 100, 0, 15) hist_ctcE = ROOT.TH1F("h_CaloTopoCluster_E", "CaloTopoCluster Energy", - 100, 0, 10) + 100, 0, 15) + hist_ecal_totE = ROOT.TH1F("h_ECalBarrelModuleThetaMergedPosition_totE", + "ECalBarrelModuleThetaMergedPosition total Energy per evt", + 100, 0, 15) + hist_ecal_posX = ROOT.TH1F("h_ECalBarrelModuleThetaMergedPosition_posX", + "ECalBarrelModuleThetaMergedPosition position X", + 150, -2770, 2770) + hist_ecal_posY = ROOT.TH1F("h_ECalBarrelModuleThetaMergedPosition_posY", + "ECalBarrelModuleThetaMergedPosition position Y", + 150, -2770, 2770) + hist_ecal_posZ = ROOT.TH1F("h_ECalBarrelModuleThetaMergedPosition_posZ", + "ECalBarrelModuleThetaMergedPosition position Z", + 150, -3100, 3100) + # loop over dataset for event in podio_reader.get("events"): @@ -41,15 +54,27 @@ def make_file(args): energy = calo.energy() hist_ctcE.Fill(energy) + energy = 0 + for ecal in event.get("ECalBarrelModuleThetaMergedPositioned"): + energy += ecal.energy() + hist_ecal_posX.Fill(ecal.position().x) + hist_ecal_posY.Fill(ecal.position().y) + hist_ecal_posZ.Fill(ecal.position().z) + hist_ecal_totE.Fill(energy) + n_evts = len(podio_reader.get("events")) if not args.no_norm: factor = 1./n_evts - hist_ccE.Scale(factor) - hist_ctcE.Scale(factor) + #hist_ccE.Scale(factor) + #hist_ctcE.Scale(factor) hist_ccE.Write() hist_ctcE.Write() + hist_ecal_totE.Write() + hist_ecal_posX.Write() + hist_ecal_posY.Write() + hist_ecal_posZ.Write() outputFile.Close() diff --git a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py index ad1337e..df42e61 100644 --- a/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py +++ b/scripts/FCCee/ALLEGRO/ALLEGRO_o1_v03/ALLEGRO_val_plots.py @@ -14,16 +14,28 @@ plot_params = { - "CaloCluster" : { + "Cluster_E;totE" : { "range" : (0, 15), "xlabel": 'Energy [MeV]', "ylabel": 'Counts / 0.15 MeV', }, - "CaloTopoCluster" : { - "range" : (0, 10), - "xlabel": 'Energy [MeV]', - "ylabel": 'Counts / 0.1 MeV', + "posX" : { + "range" : (-2800, 2800), + "xlabel": 'X [mm]', + "ylabel": 'Counts / 37 mm', + }, + + "posY" : { + "range" : (-2800, 2800), + "xlabel": 'Y [mm]', + "ylabel": 'Counts / 37 mm', + }, + + "posZ" : { + "range" : (-3150, 3150), + "xlabel": 'Z [mm]', + "ylabel": 'Counts / 41 mm', } } @@ -62,9 +74,12 @@ def make_plots(args): param = {} for p in plot_params: - if p in key: - param = plot_params[p] - break + categories = p.split(";") + for c in categories: + if c in key: + param = plot_params[p] + break + if len(param) == 0: print(f"No plot parameters found for histogram {key}!") continue From d27bf0b8085ad27bb9862b65e4af1da93d67c9f8 Mon Sep 17 00:00:00 2001 From: Enrico Lupi Date: Thu, 15 Aug 2024 14:45:33 +0200 Subject: [PATCH 31/73] Merged www and web folders from new_web_test-branch --- web/python/make_web.py | 82 ++++++++------ web/templates/main_index.html | 37 +++--- web/templates/plot_index.html | 53 ++++++++- web/templates/version_index.html | 93 +++++++++++++++ www/static/styles.css | 189 +++++++++++++++++++++++++++++++ 5 files changed, 399 insertions(+), 55 deletions(-) create mode 100644 web/templates/version_index.html create mode 100644 www/static/styles.css diff --git a/web/python/make_web.py b/web/python/make_web.py index f4c0c80..4958007 100644 --- a/web/python/make_web.py +++ b/web/python/make_web.py @@ -21,18 +21,21 @@ def __missing__(self, key): 'distributions': 'Distributions',}.items(): FOLDER_NAMES[k] = v -metadata = {} -if os.path.exists('metadata.yaml'): - with open('metadata.yaml') as f: - metadata = yaml.load(f, Loader=yaml.FullLoader) -if 'key4hep-spack' in metadata: - metadata['key4hep-spack'] = [metadata['key4hep-spack'], f"https://github.com/key4hep/key4hep-spack/commit/{metadata['key4hep-spack']}"] -if 'spack' in metadata: - metadata['spack'] = [metadata['spack'], f"https://github.com/spack/spack/commit/{metadata['spack']}"] -for k, v in metadata.items(): - if isinstance(v, str) or len(v) == 1: - metadata[k] = [v, None] -print(metadata) +def get_metadata(folder_path): + metadata = {} + file = os.path.join(folder_path, 'metadata.yaml') + if os.path.exists(file): + with open(file) as f: + metadata = yaml.load(f, Loader=yaml.FullLoader) + if 'key4hep-spack' in metadata: + metadata['key4hep-spack'] = [metadata['key4hep-spack'], f"https://github.com/key4hep/key4hep-spack/commit/{metadata['key4hep-spack']}"] + if 'spack' in metadata: + metadata['spack'] = [metadata['spack'], f"https://github.com/spack/spack/commit/{metadata['spack']}"] + for k, v in metadata.items(): + if isinstance(v, str) or len(v) == 1: + metadata[k] = [v, None] + print(metadata) + return metadata def get_latest_modified_date(folder_path): @@ -70,30 +73,43 @@ def write_plots(folder): args = parser.parse_args() -# Top level folders, maybe different types of validation -top_folders = [folder for folder in os.listdir(args.dest) if os.path.isdir(os.path.join(args.dest, folder))] -top_folders.remove('static') +# detector folders +detector_folders = [folder for folder in os.listdir(args.dest) if os.path.isdir(os.path.join(args.dest, folder))] +detector_folders.remove('static') # latest_modified_date = [get_latest_modified_date(os.path.join(args.dest, folder)) for folder in top_folders] -latest_modified_date = [get_latest_modified_date(os.path.join(args.dest, folder)) for folder in top_folders] +version_folders = [] +latest_modified_date = [] +for folder in detector_folders: + print(folder) + versions = [version_folder for version_folder in os.listdir(os.path.join(args.dest, folder)) if os.path.isdir(os.path.join(args.dest, folder, version_folder))] + version_folders.append(versions) + dates = [get_latest_modified_date(os.path.join(args.dest, folder, version)) for version in versions] + latest_modified_date.append(dates) +version_date = [[(v, d) for v, d in zip(version, date)] for version, date in zip(version_folders, latest_modified_date)] + env = jinja2.Environment(loader=jinja2.FileSystemLoader('../templates')) template = env.get_template('main_index.html') with open(os.path.join(args.dest, 'index.html'), 'w') as f: - f.write(template.render(folders=zip(top_folders, latest_modified_date))) + f.write(template.render(folders=zip(detector_folders, version_date))) # Now let's put an index.html file in each of the subdirectories and the plot folders -for folder in top_folders: - print(folder) - plot_folders = [plot_folder for plot_folder in os.listdir(os.path.join(args.dest, folder)) if os.path.isdir(os.path.join(args.dest, folder, plot_folder))] - plot_category_names = [FOLDER_NAMES[x] for x in plot_folders] - template = env.get_template('section_index.html') - with open(os.path.join(args.dest, folder, 'index.html'), 'w') as f: - f.write(template.render(plot_sections=zip(plot_folders, plot_category_names), table=metadata)) - - for plot_folder in plot_folders: - template = env.get_template('plot_index.html') - content = write_plots(os.path.join(args.dest, folder, plot_folder)) - - with open(os.path.join(args.dest, folder, plot_folder, 'index.html'), 'w') as f: - f.write(template.render(content=content)) - - +for i_folder, folder in enumerate(detector_folders): + print("Detector:", folder) + for version in version_folders[i_folder]: + print("Version:", version) + metadata = get_metadata(os.path.join(args.dest, folder, version)) + subsystem_folders = [subsyst for subsyst in os.listdir(os.path.join(args.dest, folder, version)) if os.path.isdir(os.path.join(args.dest, folder, version, subsyst))] + plot_category_names = [FOLDER_NAMES[x] for x in subsystem_folders] + + template = env.get_template('version_index.html') + with open(os.path.join(args.dest, folder, version, 'index.html'), 'w') as f: + f.write(template.render(subsystems=zip(subsystem_folders, plot_category_names), table=metadata, version=version, detector_list=zip(detector_folders, version_folders))) + + for subsystem in subsystem_folders: + print("Subsystem:", subsystem) + print("Full path:", os.path.join(args.dest, folder, version, subsystem)) + template = env.get_template('plot_index.html') + content = write_plots(os.path.join(args.dest, folder, version, subsystem)) + + with open(os.path.join(args.dest, folder, version, subsystem, 'index.html'), 'w') as f: + f.write(template.render(content=content, subsystems=subsystem_folders, version=version, subsystem=subsystem, detector_list=zip(detector_folders, version_folders))) \ No newline at end of file diff --git a/web/templates/main_index.html b/web/templates/main_index.html index 3d9b8ee..e3b67d5 100644 --- a/web/templates/main_index.html +++ b/web/templates/main_index.html @@ -6,10 +6,10 @@ Key4hep validation - + -