[TM-1719] Move beneficiaries data to demographics

app/Console/Commands/OneOff/MigrateBeneficiariesToDemographics.php

@@ -0,0 +1,198 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Models\V2\Demographics\Demographic;
+use App\Models\V2\Projects\ProjectReport;
+use Illuminate\Console\Command;
+
+class MigrateBeneficiariesToDemographics extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:migrate-beneficiaries-to-demographics';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Move beneficiaries data to demographics';
+
+    protected const ALL_MAPPING = [
+        'all' => [
+            'gender' => [
+                'male' => 'beneficiaries_men',
+                'female' => 'beneficiaries_women',
+                'non-binary' => 'beneficiaries_other',
+            ],
+            'age' => [
+                'youth' => 'beneficiaries_youth',
+                'non-youth' => 'beneficiaries_non_youth',
+            ],
+            'farmer' => [
+                'smallholder' => 'beneficiaries_smallholder',
+                'large-scale' => 'beneficiaries_large_scale',
+                'marginalized' => 'beneficiaries_scstobc_farmers',
+            ],
+            'caste' => [
+                'marginalized' => 'beneficiaries_scstobc',
+            ],
+            'total' => ['beneficiaries', 'total_community_partners'],
+        ],
+    ];
+
+    protected const TRAINING_MAPPING = [
+        'training' => [
+            'gender' => [
+                'male' => 'beneficiaries_training_men',
+                'female' => 'beneficiaries_training_women',
+                'non-binary' => 'beneficiaries_training_other',
+            ],
+            'age' => [
+                'youth' => 'beneficiaries_training_youth',
+                'non-youth' => 'beneficiaries_training_non_youth',
+            ],
+            'total' => 'beneficiaries_skills_knowledge_increase',
+        ],
+    ];
+
+    protected const MIGRATION_MAPPING = [
+        'all-beneficiaries' => self::ALL_MAPPING,
+        'training-beneficiaries' => self::TRAINING_MAPPING,
+    ];
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->info('Moving project report beneficiaries data to Demographics...');
+        $this->withProgressBar(ProjectReport::count(), function ($progressBar) {
+            ProjectReport::chunkById(100, function ($projectReports) use ($progressBar) {
+                foreach ($projectReports as $projectReport) {
+                    $this->convertJobs($projectReport);
+                    $progressBar->advance();
+                }
+            });
+        });
+
+        $this->info("\n\nCompleted moving project report beneficiaries data to Demographics.");
+    }
+
+    private function convertJobs(ProjectReport $projectReport): void
+    {
+        foreach (self::MIGRATION_MAPPING as $demographicType => $mapping) {
+            foreach ($mapping as $collection => $types) {
+                /** @var Demographic $demographic */
+                $demographic = null;
+                foreach ($types as $type => $subtypes) {
+                    if ($type == 'total') {
+                        $fields = is_array($subtypes) ? $subtypes : [$subtypes];
+                        // Make sure gender / age demographics are balanced and reach at least to the "_total" field
+                        // for this type of job from the original report. Pad gender and age demographics with an
+                        // "unknown" if needed.
+                        $genderTotal = $demographic?->entries()->gender()->sum('amount') ?? 0;
+                        $ageTotal = $demographic?->entries()->age()->sum('amount') ?? 0;
+
+                        $totals = [$genderTotal, $ageTotal];
+                        foreach ($fields as $field) {
+                            $totals[] = $projectReport[$field];
+                        }
+                        $targetTotal = max($totals);
+
+                        if ($demographic == null && $targetTotal > 0) {
+                            $demographic = $projectReport->demographics()->create([
+                                'type' => $demographicType,
+                                'collection' => $collection,
+                                'hidden' => false,
+                            ]);
+                        }
+
+                        if ($genderTotal < $targetTotal) {
+                            $demographic->entries()->create([
+                                'type' => 'gender',
+                                'subtype' => 'unknown',
+                                'amount' => $targetTotal - $genderTotal,
+                            ]);
+                        }
+                        if ($ageTotal < $targetTotal) {
+                            $demographic->entries()->create([
+                                'type' => 'age',
+                                'subtype' => 'unknown',
+                                'amount' => $targetTotal - $ageTotal,
+                            ]);
+                        }
+                    } else {
+                        // If none of the fields for this type exist, skip
+                        $fields = collect(array_values($subtypes));
+                        if ($fields->first(fn ($field) => $projectReport[$field] > 0) == null) {
+                            continue;
+                        }
+
+                        if ($demographic == null) {
+                            $demographic = $projectReport->demographics()->create([
+                                'type' => $demographicType,
+                                'collection' => $collection,
+                                'hidden' => false,
+                            ]);
+                        }
+                        foreach ($subtypes as $subtype => $field) {
+                            $value = $projectReport[$field];
+                            if ($value > 0) {
+                                $demographic->entries()->create([
+                                    'type' => $type,
+                                    'subtype' => $subtype,
+                                    'amount' => $value,
+                                ]);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        if ($projectReport->trainingBeneficiariesTotal > $projectReport->allBeneficiariesTotal) {
+            // in this case, the training data had a greater gender total than the "all" gender total, so we want
+            // to pad "all" so that they're equal.
+            $padValue = $projectReport->trainingBeneficiariesTotal - $projectReport->allBeneficiariesTotal;
+
+            $all = $projectReport->allBeneficiaries()->first();
+            if ($all == null) {
+                $all = $projectReport->demographics()->create([
+                    'type' => Demographic::ALL_BENEFICIARIES_TYPE,
+                    'collection' => 'all',
+                    'hidden' => false,
+                ]);
+            }
+
+            // We can assume that gender / age have already been balanced and just add the pad value to both
+            $gender = $all->entries()->where(['type' => 'gender', 'subtype' => 'unknown'])->first();
+            if ($gender == null) {
+                $all->entries()->create([
+                    'type' => 'gender',
+                    'subtype' => 'unknown',
+                    'amount' => $padValue,
+                ]);
+            } else {
+                $gender->amount += $padValue;
+                $gender->save();
+            }
+
+            $age = $all->entries()->where(['type' => 'age', 'subtype' => 'unknown'])->first();
+            if ($age == null) {
+                $all->entries()->create([
+                    'type' => 'age',
+                    'subtype' => 'unknown',
+                    'amount' => $padValue,
+                ]);
+            } else {
+                $age->amount += $padValue;
+                $age->save();
+            }
+        }
+    }
+}

app/Exports/V2/BaseExportFormSubmission.php

@@ -72,13 +72,14 @@ protected function getAnswer(array $field, array $answers, ?string $frameworkKey
                 case 'restorationPartners':
                 case 'jobs':
                 case 'volunteers':
+                case 'beneficiaries':
                     $list = [];
                     $demographic = $answer->first();
                     if ($demographic == null) {
                         return '';
                     }
 
-                    $types = ['gender' => [], 'age' => [], 'ethnicity' => [], 'caste' => []];
+                    $types = ['gender' => [], 'age' => [], 'ethnicity' => [], 'farmer' => [], 'caste' => []];
                     foreach ($demographic->entries as $entry) {
                         $value = match ($entry->type) {
                             'ethnicity' => [$entry->amount, $entry->subtype, $entry->name],
@@ -88,6 +89,9 @@ protected function getAnswer(array $field, array $answers, ?string $frameworkKey
                     }
                     $list[] = 'gender:(' . implode(')(', $types['gender']) . ')';
                     $list[] = 'age:(' . implode(')(', $types['age']) . ')';
+                    if ($demographic->type == 'beneficiaries' && $demographic->collection == 'all') {
+                        $list[] = 'farmer:(' . implode(')(', $types['farmer']) . ')';
+                    }
                     if ($frameworkKey == 'hbf') {
                         $list[] = 'caste:(' . implode(')(', $types['caste']) . ')';
                     } elseif ($field['input_type'] == 'workdays' || $field['input_type'] == 'restorationPartners') {

app/Http/Resources/V2/Demographics/DemographicResource.php

@@ -11,7 +11,6 @@ public function toArray($request)
         return [
             'uuid' => $this->uuid,
             'collection' => $this->collection,
-            'readable_collection' => $this->readable_collection,
             'demographics' => empty($this->entries) ? [] : DemographicEntryResource::collection($this->entries),
         ];
     }

app/Http/Resources/V2/ProjectReports/ProjectReportResource.php

@@ -52,17 +52,8 @@ public function toArray($request)
             'planted_trees' => $this->planted_trees,
             'new_jobs_description' => $this->new_jobs_description,
             'volunteers_work_description' => $this->volunteers_work_description,
-            'beneficiaries' => $this->beneficiaries,
             'beneficiaries_description' => $this->beneficiaries_description,
-            'beneficiaries_women' => $this->beneficiaries_women,
-            'beneficiaries_men' => $this->beneficiaries_men,
-            'beneficiaries_non_youth' => $this->beneficiaries_non_youth,
-            'beneficiaries_youth' => $this->beneficiaries_youth,
-            'beneficiaries_smallholder' => $this->beneficiaries_smallholder,
-            'beneficiaries_large_scale' => $this->beneficiaries_large_scale,
-            'beneficiaries_income_increase' => $this->beneficiaries_income_increase,
             'beneficiaries_income_increase_description' => $this->beneficiaries_income_increase_description,
-            'beneficiaries_skills_knowledge_increase' => $this->beneficiaries_skills_knowledge_increase,
             'beneficiaries_skills_knowledge_increase_description' => $this->beneficiaries_skills_knowledge_increase_description,
             'organisation' => new OrganisationLiteResource($this->organisation),
             'project' => new ProjectLiteResource($this->project),
@@ -93,22 +84,13 @@ public function toArray($request)
             'convergence_schemes' => $this->convergence_schemes,
             'convergence_amount' => $this->convergence_amount,
             'community_partners_assets_description' => $this->community_partners_assets_description,
-            'beneficiaries_scstobc_farmers' => $this->beneficiaries_scstobc_farmers,
-            'beneficiaries_scstobc' => $this->beneficiaries_scstobc,
             'people_knowledge_skills_increased' => $this->people_knowledge_skills_increased,
             'indirect_beneficiaries' => $this->indirect_beneficiaries,
             'indirect_beneficiaries_description' => $this->indirect_beneficiaries_description,
             'workdays_direct_total' => $this->workdays_direct_total,
             'workdays_convergence_total' => $this->workdays_convergence_total,
             'non_tree_total' => $this->non_tree_total,
-            'total_community_partners' => $this->total_community_partners,
             'business_milestones' => $this->business_milestones,
-            'beneficiaries_other' => $this->beneficiaries_other,
-            'beneficiaries_training_women' => $this->beneficiaries_training_women,
-            'beneficiaries_training_men' => $this->beneficiaries_training_men,
-            'beneficiaries_training_other' => $this->beneficiaries_training_other,
-            'beneficiaries_training_youth' => $this->beneficiaries_training_youth,
-            'beneficiaries_training_non_youth' => $this->beneficiaries_training_non_youth,
         ];
 
         return $this->appendFilesToResource($data);

app/Models/Traits/HasDemographics.php

@@ -18,7 +18,9 @@ trait HasDemographics
         'indirectRestorationPartners' => ['type' => Demographic::RESTORATION_PARTNER_TYPE, 'collections' => 'indirect'],
         'jobsFullTimeTotal' => ['type' => Demographic::JOBS_TYPE, 'collections' => 'full-time'],
         'jobsPartTimeTotal' => ['type' => Demographic::JOBS_TYPE, 'collections' => 'part-time'],
-        'volunteersTotal' => ['type' => Demographic::VOLUNTEERS_TYPE, 'collections' => 'volunteer'],
+        'volunteersTotal' => ['type' => Demographic::VOLUNTEERS_TYPE],
+        'allBeneficiariesTotal' => ['type' => Demographic::ALL_BENEFICIARIES_TYPE],
+        'trainingBeneficiariesTotal' => ['type' => Demographic::TRAINING_BENEFICIARIES_TYPE],
     ];
 
     public static function bootHasDemographics()
@@ -47,19 +49,22 @@ public static function bootHasDemographics()
                     $collectionSets['full-time'],
                     $collectionSets['part-time'],
                 ])->flatten(),
-                Demographic::VOLUNTEERS_TYPE => collect([
-                    $collectionSets['volunteer'],
-                ])->flatten(),
+                // These three define a single collection each, and simply rely on the type level relation above
+                Demographic::VOLUNTEERS_TYPE,
+                Demographic::ALL_BENEFICIARIES_TYPE,
+                Demographic::TRAINING_BENEFICIARIES_TYPE => null,
                 default => throw new InternalErrorException("Unrecognized demographic type: $demographicType"),
             };
-            $collections->each(function ($collection) use ($attributePrefix) {
-                self::resolveRelationUsing(
-                    $attributePrefix . Str::studly($collection),
-                    function ($entity) use ($attributePrefix, $collection) {
-                        return $entity->$attributePrefix()->collection($collection);
-                    }
-                );
-            });
+            if (! empty($collections)) {
+                $collections->each(function ($collection) use ($attributePrefix) {
+                    self::resolveRelationUsing(
+                        $attributePrefix . Str::studly($collection),
+                        function ($entity) use ($attributePrefix, $collection) {
+                            return $entity->$attributePrefix()->collection($collection);
+                        }
+                    );
+                });
+            }
         });
     }
 
@@ -79,7 +84,9 @@ public function getAttribute($key)
         if (array_key_exists($keyNormalized, self::DEMOGRAPHIC_ATTRIBUTES)) {
             $definition = self::DEMOGRAPHIC_ATTRIBUTES[$keyNormalized];
             $type = $definition['type'];
-            $collections = self::DEMOGRAPHIC_COLLECTIONS[$type][$definition['collections']];
+            $collections = is_string(self::DEMOGRAPHIC_COLLECTIONS[$type])
+                ? [self::DEMOGRAPHIC_COLLECTIONS[$type]]
+                : self::DEMOGRAPHIC_COLLECTIONS[$type][$definition['collections']];
 
             return $this->sumTotalDemographicAmounts(Str::camel($type), $collections);
         }

app/Models/Traits/UsesLinkedFields.php

@@ -266,6 +266,7 @@ private function syncRelation(string $property, string $inputType, $data, bool $
                 'restorationPartners',
                 'jobs',
                 'volunteers',
+                'beneficiaries',
                 'stratas',
                 'invasive',
                 'seedings',