Merge branch 'master' into kmt_configWarning

.github/workflows/deploy-dtds.yaml

@@ -1,18 +1,20 @@
 name: deploy-dtds-on-website
 
 on:
-  push:
-    branches:
-      - master
+  pull_request:
+    types:
+      - closed
     paths:
-      - matsim/src/main/resources/dtd
+      - 'matsim/src/main/resources/dtd/**'
 
 jobs:
   rsync-dtds:
+    if: github.event.pull_request.merged == true  # only if PR closed by merging
     name: sync DTDs to website
     runs-on: ubuntu-latest
 
     steps:
+      - uses: actions/checkout@v4
       - name: rsync dtds
         uses: burnett01/[email protected]
         with:

.github/workflows/full-integration.yaml

@@ -14,7 +14,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest]
+        #os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
       - name: Prepare git

contribs/application/pom.xml

@@ -87,7 +87,7 @@
 		<dependency>
 			<groupId>com.github.matsim-org</groupId>
 			<artifactId>gtfs2matsim</artifactId>
-			<version>fc8b13954d</version>
+			<version>47b0802a29</version>
 			<exclusions>
 				<!-- Exclude unneeded dependencies and these with known CVE -->
 				<exclusion>

contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java

@@ -446,11 +446,11 @@ public static Path matchInput(String name, Path dir) {
 			return path.get();
 
 		// Even more permissive pattern
-		path = matchPattern(".+\\.[a-zA-Z0-9_.\\-]*(_|\\.)" + name + "\\..+", dir);
+		path = matchPattern(".+[a-zA-Z0-9_.\\-]*(_|\\.)" + name + ".+", dir);
 		if (path.isPresent())
 			return path.get();
 
-		throw new IllegalArgumentException("Could not match input file: " + name);
+		throw new IllegalArgumentException("Could not match input file: %s (in %s)".formatted(name, dir));
 	}
 
 	private static Optional<Path> matchSuffix(String suffix, Path dir) {

contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java

@@ -7,6 +7,7 @@
 import org.locationtech.jts.geom.Envelope;
 import org.matsim.api.core.v01.Coord;
 import org.matsim.api.core.v01.Scenario;
+import org.matsim.api.core.v01.TransportMode;
 import org.matsim.application.ApplicationUtils;
 import org.matsim.application.CommandSpec;
 import org.matsim.application.MATSimAppCommand;
@@ -20,6 +21,7 @@
 import org.matsim.core.config.Config;
 import org.matsim.core.config.ConfigUtils;
 import org.matsim.core.scenario.ScenarioUtils;
+import org.matsim.core.utils.collections.CollectionUtils;
 import org.matsim.core.utils.geometry.CoordinateTransformation;
 import org.matsim.core.utils.io.IOUtils;
 import picocli.CommandLine;
@@ -28,10 +30,7 @@
 import java.nio.file.Path;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 
 @CommandLine.Command(
 	name = "noise-analysis",
@@ -67,7 +66,7 @@ public class NoiseAnalysis implements MATSimAppCommand {
 
 	@CommandLine.Option(names = "--consider-activities", split = ",", description = "Considered activities for noise calculation." +
 		" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "home*,work*,educ*,leisure*")
-	private Set<String> considerActivities;
+	private Set<String> consideredActivities;
 
 	@CommandLine.Option(names = "--noise-barrier", description = "Path to the noise barrier File", defaultValue = "")
 	private String noiseBarrierFile;
@@ -87,10 +86,19 @@ public Integer call() throws Exception {
 		boolean overrideParameters = ! ConfigUtils.hasModule(config, NoiseConfigGroup.class);
 		NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class);
 
-		if(overrideParameters){
-			log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file!");
-			noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
-			noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));
+		if (overrideParameters){
+			log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file and the output_config.xml!");
+			noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities.toArray(String[]::new));
+			noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities.toArray(String[]::new));
+
+			{
+				//the default settings are now actually the same as what we 'override' here, but let's leave it here for clarity.
+				Set<String> ignoredNetworkModes = CollectionUtils.stringArrayToSet( new String[]{TransportMode.bike, TransportMode.walk, TransportMode.transit_walk, TransportMode.non_network_walk} );
+				noiseParameters.setNetworkModesToIgnoreSet( ignoredNetworkModes );
+
+				String[] hgvIdPrefixes = {"lkw", "truck", "freight"};
+				noiseParameters.setHgvIdPrefixesArray( hgvIdPrefixes );
+			}
 
 			//use actual speed and not freespeed
 			noiseParameters.setUseActualSpeedLevel(true);

contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java

@@ -4,10 +4,7 @@
 import it.unimi.dsi.fastutil.ints.IntList;
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import it.unimi.dsi.fastutil.ints.IntSet;
-import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.objects.Object2IntMap;
-import it.unimi.dsi.fastutil.objects.Object2LongMap;
-import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import it.unimi.dsi.fastutil.objects.*;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.math3.analysis.interpolation.LoessInterpolator;
@@ -27,6 +24,7 @@
 import org.matsim.core.utils.io.IOUtils;
 import picocli.CommandLine;
 import tech.tablesaw.api.*;
+import tech.tablesaw.columns.strings.AbstractStringColumn;
 import tech.tablesaw.io.csv.CsvReadOptions;
 import tech.tablesaw.joining.DataFrameJoiner;
 import tech.tablesaw.selection.Selection;
@@ -37,6 +35,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static tech.tablesaw.aggregate.AggregateFunctions.count;
@@ -46,8 +45,9 @@
 	requires = {"trips.csv", "persons.csv"},
 	produces = {
 		"mode_share.csv", "mode_share_per_dist.csv", "mode_users.csv", "trip_stats.csv",
-		"mode_share_per_%s.csv", "population_trip_stats.csv", "trip_purposes_by_hour.csv",
-		"mode_share_distance_distribution.csv", "mode_shift.csv",
+		"mode_share_per_purpose.csv", "mode_share_per_%s.csv",
+		"population_trip_stats.csv", "trip_purposes_by_hour.csv",
+		"mode_share_distance_distribution.csv", "mode_shift.csv", "mode_chains.csv",
 		"mode_choices.csv", "mode_choice_evaluation.csv", "mode_choice_evaluation_per_mode.csv",
 		"mode_confusion_matrix.csv", "mode_prediction_error.csv"
 	}
@@ -283,10 +283,15 @@ public Integer call() throws Exception {
 
 		joined.addColumns(dist_group);
 
+		TextColumn purpose = joined.textColumn("end_activity_type");
+
+		// Remove suffix durations like _345
+		purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", ""));
+
 		writeModeShare(joined, labels);
 
 		if (groups != null) {
-			groups.analyzeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g));
+			groups.writeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g));
 		}
 
 		if (persons.containsColumn(ATTR_REF_MODES)) {
@@ -305,17 +310,24 @@ public Integer call() throws Exception {
 
 		writePopulationStats(persons, joined);
 
-		writeTripStats(joined);
-
-		writeTripPurposes(joined);
-
-		writeTripDistribution(joined);
-
-		writeModeShift(joined);
+		tryRun(this::writeTripStats, joined);
+		tryRun(this::writeTripPurposes, joined);
+		tryRun(this::writeTripDistribution, joined);
+		tryRun(this::writeModeShift, joined);
+		tryRun(this::writeModeChains, joined);
+		tryRun(this::writeModeStatsPerPurpose, joined);
 
 		return 0;
 	}
 
+	private void tryRun(ThrowingConsumer<Table> f, Table df) {
+		try {
+			f.accept(df);
+		} catch (IOException e) {
+			log.error("Error while running method", e);
+		}
+	}
+
 	private void writeModeShare(Table trips, List<String> labels) {
 
 		Table aggr = trips.summarize("trip_id", count).by("dist_group", "main_mode");
@@ -502,11 +514,6 @@ private void writeTripPurposes(Table trips) {
 			IntColumn.create("arrival_h", arrival.intStream().toArray())
 		);
 
-		TextColumn purpose = trips.textColumn("end_activity_type");
-
-		// Remove suffix durations like _345
-		purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", ""));
-
 		Table tArrival = trips.summarize("trip_id", count).by("end_activity_type", "arrival_h");
 
 		tArrival.column(0).setName("purpose");
@@ -610,6 +617,89 @@ private void writeModeShift(Table trips) throws IOException {
 		aggr.write().csv(output.getPath("mode_shift.csv").toFile());
 	}
 
+	/**
+	 * Collects information about all modes used during one day.
+	 */
+	private void writeModeChains(Table trips) throws IOException {
+
+		Map<String, List<String>> modesPerPerson = new LinkedHashMap<>();
+
+		for (Row trip : trips) {
+			String id = trip.getString("person");
+			String mode = trip.getString("main_mode");
+			modesPerPerson.computeIfAbsent(id, s -> new LinkedList<>()).add(mode);
+		}
+
+		// Store other values explicitly
+		ObjectDoubleMutablePair<String> other = ObjectDoubleMutablePair.of("other", 0);
+		Object2DoubleMap<String> chains = new Object2DoubleOpenHashMap<>();
+		for (List<String> modes : modesPerPerson.values()) {
+			String key;
+			if (modes.size() == 1)
+				key = modes.getFirst();
+			else if (modes.size() > 6) {
+				other.right(other.rightDouble() + 1);
+				continue;
+			} else
+				key = String.join("-", modes);
+
+			chains.mergeDouble(key, 1, Double::sum);
+		}
+
+
+		List<ObjectDoubleMutablePair<String>> counts = chains.object2DoubleEntrySet().stream()
+			.map(e -> ObjectDoubleMutablePair.of(e.getKey(), (int) e.getDoubleValue()))
+			.sorted(Comparator.comparingDouble(p -> -p.rightDouble()))
+			.collect(Collectors.toList());
+
+		// Aggregate entries to prevent file from getting too large
+		for (int i = 250; i < counts.size(); i++) {
+			other.right(other.rightDouble() + counts.get(i).rightDouble());
+		}
+		counts = counts.subList(0, Math.min(counts.size(), 250));
+		counts.add(other);
+
+		counts.sort(Comparator.comparingDouble(p -> -p.rightDouble()));
+
+
+		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mode_chains.csv")), CSVFormat.DEFAULT)) {
+
+			printer.printRecord("modes", "count", "share");
+
+			double total = counts.stream().mapToDouble(ObjectDoubleMutablePair::rightDouble).sum();
+			for (ObjectDoubleMutablePair<String> p : counts) {
+				printer.printRecord(p.left(), (int) p.rightDouble(), p.rightDouble() / total);
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private void writeModeStatsPerPurpose(Table trips) {
+
+		Table aggr = trips.summarize("trip_id", count).by("end_activity_type", "main_mode");
+
+		Comparator<Row> cmp = Comparator.comparing(row -> row.getString("end_activity_type"));
+		aggr = aggr.sortOn(cmp.thenComparing(row -> row.getString("main_mode")));
+
+		aggr.doubleColumn(aggr.columnCount() - 1).setName("share");
+		aggr.column("end_activity_type").setName("purpose");
+
+		Set<String> purposes = (Set<String>) aggr.column("purpose").asSet();
+
+		// Norm each purpose to 1
+		// It was not clear if the purpose is a string or text colum, therefor this code uses the abstract version
+		for (String label : purposes) {
+			DoubleColumn all = aggr.doubleColumn("share");
+			Selection sel = ((AbstractStringColumn<?>) aggr.column("purpose")).isEqualTo(label);
+
+			double total = all.where(sel).sum();
+			if (total > 0)
+				all.set(sel, all.divide(total));
+		}
+
+		aggr.write().csv(output.getPath("mode_share_per_purpose.csv").toFile());
+	}
+
 	/**
 	 * How shape file filtering should be applied.
 	 */
@@ -619,4 +709,9 @@ enum LocationFilter {
 		home,
 		none
 	}
+
+	@FunctionalInterface
+	private interface ThrowingConsumer<T> {
+		void accept(T t) throws IOException;
+	}
 }

contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java

@@ -100,7 +100,7 @@ final class TripByGroupAnalysis {
 		}
 	}
 
-	void analyzeModeShare(Table trips, List<String> dists, List<String> modeOrder, Function<String, Path> output) {
+	void writeModeShare(Table trips, List<String> dists, List<String> modeOrder, Function<String, Path> output) {
 
 		for (Group group : groups) {