TAN-3906 - User fields in survey form

back/app/controllers/web_api/v1/ideas_controller.rb

@@ -147,6 +147,12 @@ def draft_by_phase
     draft_idea =
       (current_user && Idea.find_by(creation_phase_id: params[:phase_id], author: current_user, publication_status: 'draft')) ||
       Idea.new(project: phase.project, author: current_user, publication_status: 'draft')
+
+    # merge custom field values from the user's profile, if they exist
+    if phase.user_fields_in_form
+      user_values = current_user&.custom_field_values&.transform_keys { |key| "u_#{key}" }
+      draft_idea.custom_field_values = user_values.merge(draft_idea.custom_field_values) if current_user
+    end
     render_show draft_idea, check_auth: false
   end
 

back/app/controllers/web_api/v1/phases_controller.rb

@@ -138,6 +138,7 @@ def phase_params
       :reacting_threshold,
       :expire_days_limit,
       :manual_voters_amount,
+      :user_fields_in_form,
       {
         title_multiloc: CL2_SUPPORTED_LOCALES,
         description_multiloc: CL2_SUPPORTED_LOCALES,

back/app/models/custom_field.rb

@@ -201,7 +201,7 @@ def required?
   end
 
   def domicile?
-    key == 'domicile' && code == 'domicile'
+    (key == 'domicile' && code == 'domicile') || key == 'u_domicile'
   end
 
   def file_upload?

back/app/serializers/web_api/v1/phase_serializer.rb

@@ -15,7 +15,7 @@ class WebApi::V1::PhaseSerializer < WebApi::V1::BaseSerializer
   %i[
     voting_method voting_max_total voting_min_total
     voting_max_votes_per_idea baskets_count
-    native_survey_title_multiloc native_survey_button_multiloc
+    user_fields_in_form native_survey_title_multiloc native_survey_button_multiloc
     expire_days_limit reacting_threshold autoshare_results_enabled
   ].each do |attribute_name|
     attribute attribute_name, if: proc { |phase|

back/app/services/idea_custom_fields_service.rb

@@ -51,7 +51,8 @@ def importable_fields
   end
 
   def enabled_fields
-    all_fields.select(&:enabled?)
+    fields = all_fields.select(&:enabled?)
+    add_user_fields(fields)
   end
 
   def enabled_fields_with_other_options
@@ -198,5 +199,51 @@ def section1_title?(field, attribute)
     field.code == 'ideation_section1' && attribute == :title_multiloc
   end
 
+  def add_user_fields(fields)
+    phase = @custom_form.participation_context
+    return fields unless @participation_method.supports_survey_form? && phase.user_fields_in_form
+
+    last_page = fields.last
+    fields.delete(last_page)
+
+    # Get the user fields from the permission (returns platform defaults if they don't exist)
+    user_requirements_service = Permissions::UserRequirementsService.new
+    permission = phase.permissions.find_by(action: 'posting_idea')
+    user_fields = user_requirements_service.requirements_custom_fields(permission)
+
+    # TODO: Hide any user fields that are locked for the user through the verification method
+
+    # Transform the user fields to pretend to be idea fields
+    user_fields.each do |field|
+      field.dropdown_layout = true if field.dropdown_layout_type?
+      field.code = nil # Remove the code so it doesn't appear as built in
+      field.key = "u_#{field.key}" # Change the key so we cans clearly identify user data in the saved data
+      field.resource = custom_form # User field pretend to be part of the form
+    end
+
+    user_page = CustomField.new(
+      id: SecureRandom.uuid,
+      key: 'user_page',
+      title_multiloc: { 'en' => 'About you' },
+      resource: custom_form,
+      input_type: 'page',
+      page_layout: 'default'
+    )
+
+    # Change any logic end pages to reference the user page instead
+    fields.each do |field|
+      if field.logic['rules']
+        field.logic['rules'].map! do |rule|
+          rule['goto_page_id'] = user_page.id if rule['goto_page_id'] == last_page.id
+          rule
+        end
+      elsif field.logic['next_page_id'] == last_page.id
+        field.logic['next_page_id'] = user_page.id
+      end
+    end
+
+    fields + [user_page] + user_fields + [last_page]
+  end
+
   attr_reader :custom_form, :participation_method
 end

back/app/services/json_schema_generator_service.rb

@@ -72,10 +72,24 @@ def visit_select(field)
     {
       type: 'string'
     }.tap do |json|
-      options = field.ordered_options
+      if field.domicile?
+        areas = Area.order(:ordering).map do |area|
+          {
+            const: area.id,
+            title: multiloc_service.t(area.title_multiloc)
+          }
+        end
+        areas.push({
+          const: 'outside',
+          title: I18n.t('custom_field_options.domicile.outside')
+        })
+        json[:enum] = areas.pluck(:const)
+      else
+        options = field.ordered_options
 
-      unless options.empty?
-        json[:enum] = options.map(&:key)
+        unless options.empty?
+          json[:enum] = options.map(&:key)
+        end
       end
     end
   end

back/app/services/permissions/permissions_custom_fields_service.rb

@@ -5,7 +5,8 @@ class PermissionsCustomFieldsService
     def fields_for_permission(permission, return_hidden: false)
       # Currently we don't support custom fields for 'everyone' though the rest of the code is ready for it
       # So we have added this block until we do support it
-      return [] if permission.permitted_by == 'everyone'
+      # TODO: JS - make this dependent on whether the phase has inline questions set or not
+      return [] if permission.permitted_by == 'everyone' && !permission.permission_scope&.user_fields_in_form
 
       fields = if permission.global_custom_fields
         default_fields(permission)

back/app/services/permissions/user_requirements_service.rb

@@ -66,11 +66,15 @@ def base_requirements(permission)
         missing_user_attributes: %i[first_name last_name email confirmation password]
       },
       verification: false,
-      custom_fields: requirements_custom_fields(permission).to_h { |field| [field.key, (field.required ? 'required' : 'optional')] },
+      custom_fields: [],
       onboarding: onboarding_possible?,
       group_membership: @check_groups_and_verification && permission.groups.any?
     }
 
+    unless permission.permission_scope&.user_fields_in_form
+      users_requirements[:custom_fields] = requirements_custom_fields(permission).to_h { |field| [field.key, (field.required ? 'required' : 'optional')] }
+    end
+
     everyone_confirmed_email_requirements = users_requirements.deep_dup.tap do |requirements|
       requirements[:authentication][:missing_user_attributes] = %i[email confirmation]
       requirements[:onboarding] = false

back/app/services/side_fx_idea_service.rb

@@ -28,6 +28,7 @@ def after_create(idea, user)
     after_submission idea, user if idea.submitted_or_published?
     after_publish idea, user if idea.published?
     enqueue_embeddings_job(idea)
+    update_user_profile(idea, user)
 
     log_activities_if_cosponsors_added(idea, user)
   end
@@ -40,6 +41,7 @@ def before_update(idea, user)
     before_publish_or_submit idea, user if idea.will_be_submitted? || idea.will_be_published?
   end
 
+  # rubocop:disable Metrics/MethodLength
   def after_update(idea, user)
     # We need to check if the idea was just submitted or just published before
     # we do anything else because updates to the idea can change this state.
@@ -100,6 +102,8 @@ def after_update(idea, user)
       )
     end
 
+    update_user_profile(idea, user)
+
     enqueue_embeddings_job(idea) if idea.title_multiloc_previously_changed? || idea.body_multiloc_previously_changed?
 
     if idea.manual_votes_amount_previously_changed?
@@ -114,6 +118,7 @@ def after_update(idea, user)
 
     log_activities_if_cosponsors_added(idea, user)
   end
+  # rubocop:enable Metrics/MethodLength
 
   def after_destroy(frozen_idea, user)
     frozen_idea.phases.each(&:update_manual_votes_count!) if frozen_idea.manual_votes_amount.present?
@@ -215,6 +220,14 @@ def log_activities_if_cosponsors_added(idea, user)
   def enqueue_embeddings_job(idea)
     UpsertEmbeddingJob.perform_later(idea) if AppConfiguration.instance.feature_activated?('similar_inputs')
   end
+
+  # update the user profile if user fields are changed as part of a survey
+  def update_user_profile(idea, user)
+    return unless user && idea.creation_phase&.user_fields_in_form
+
+    user_values_from_idea = idea.custom_field_values.select { |key, _value| key.start_with?('u_') }.transform_keys { |key| key[2..] }
+    user.update!(custom_field_values: user.custom_field_values.merge(user_values_from_idea))
+  end
 end
 
 SideFxIdeaService.prepend(FlagInappropriateContent::Patches::SideFxIdeaService)

back/app/services/survey_results_generator_service.rb

@@ -235,6 +235,8 @@ def responses_to_geographic_input_type(field)
   end
 
   def build_select_response(answers, field)
+
+
     # TODO: This is an additional query for selects so performance issue here
     question_response_count = inputs.where("custom_field_values->'#{field.key}' IS NOT NULL").count
 
@@ -262,6 +264,10 @@ def get_option_multilocs(field)
       return build_scaled_input_multilocs(field)
     end
 
+    if field.domicile?
+      return build_domicile_multilocs(field)
+    end
+
     field.options.each_with_object({}) do |option, accu|
       option_detail = { title_multiloc: option.title_multiloc }
       option_detail[:image] = option.image&.image&.versions&.transform_values(&:url) if field.support_option_images?
@@ -287,6 +293,13 @@ def build_scaled_input_multilocs(field)
     answer_multilocs
   end
 
+  def build_domicile_multilocs(field)
+    field.options.each_with_object({}) do |option, accu|
+      option_detail = { title_multiloc: option.area&.title_multiloc || MultilocService.new.i18n_to_multiloc('custom_field_options.domicile.outside') }
+      accu[option.area&.id || 'outside'] = option_detail
+    end
+  end
+
   def get_option_logic(field)
     return {} if field.logic.blank?
 
@@ -390,6 +403,8 @@ def group_query(query, group: false)
   end
 
   def generate_answer_keys(field)
+    return field.options.map { |o| o.area&.id || 'outside' } + [nil] if field.domicile?
+
     (%w[linear_scale rating].include?(field.input_type) ? (1..field.maximum).to_a : field.options.map(&:key)) + [nil]
   end
 

back/app/services/user_json_schema_generator_service.rb

@@ -9,22 +9,4 @@ def visit_number(field)
       schema[:oneOf] = years.map { |y| { const: y } }
     end
   end
-
-  def visit_select(field)
-    return super unless field.code == 'domicile'
-
-    super.tap do |schema|
-      areas = Area.order(:ordering).map do |area|
-        {
-          const: area.id,
-          title: multiloc_service.t(area.title_multiloc)
-        }
-      end
-      areas.push({
-        const: 'outside',
-        title: I18n.t('custom_field_options.domicile.outside')
-      })
-      schema[:enum] = areas.pluck(:const)
-    end
-  end
 end

back/config/schemas/settings.schema.json.erb

@@ -1353,6 +1353,18 @@
         }
       },
 
+      "user_fields_in_surveys": {
+        "type": "object",
+        "title": "User fields in surveys",
+        "description": "Allow surveys to include user fields in a survey form.",
+        "additionalProperties": false,
+        "required": ["allowed", "enabled"],
+        "properties": {
+          "allowed": { "type": "boolean", "default": false },
+          "enabled": { "type": "boolean", "default": false }
+        }
+      },
+
       "verification": {
         "type": "object",
         "title": "Verification",

back/db/migrate/20250220161323_add_user_fields_in_form_to_phase.rb

@@ -0,0 +1,5 @@
+class AddUserFieldsInFormToPhase < ActiveRecord::Migration[7.1]
+  def change
+    add_column :phases, :user_fields_in_form, :boolean, default: false, null: false
+  end
+end

back/db/structure.sql

@@ -1564,7 +1564,8 @@ CREATE TABLE public.phases (
     manual_votes_count integer DEFAULT 0 NOT NULL,
     manual_voters_amount integer,
     manual_voters_last_updated_by_id uuid,
-    manual_voters_last_updated_at timestamp(6) without time zone
+    manual_voters_last_updated_at timestamp(6) without time zone,
+    user_fields_in_form boolean DEFAULT false NOT NULL
 );
 
 
@@ -6826,6 +6827,7 @@ ALTER TABLE ONLY public.ideas_topics
 SET search_path TO public,shared_extensions;
 
 INSERT INTO "schema_migrations" (version) VALUES
+('20250220161323'),
 ('20250204143605'),
 ('20250120125531'),
 ('20250117121004'),

back/engines/commercial/multi_tenancy/db/seeds/tenants.rb

@@ -479,6 +479,10 @@ def create_localhost_tenant
             project_library: {
               enabled: false,
               allowed: false
+            },
+            user_fields_in_surveys: {
+              enabled: true,
+              allowed: true
             }
           })
         )

back/lib/participation_method/native_survey.rb

@@ -118,7 +118,7 @@ def supports_permitted_by_everyone?
     end
 
     def supports_serializing?(attribute)
-      %i[native_survey_title_multiloc native_survey_button_multiloc].include?(attribute)
+      %i[user_fields_in_form native_survey_title_multiloc native_survey_button_multiloc].include?(attribute)
     end
 
     def supports_submission?

back/spec/services/idea_custom_fields_service_spec.rb

@@ -241,6 +241,50 @@
     end
   end
 
+  context 'survey form with user fields' do
+    describe 'enabled_fields' do
+      let!(:custom_form) { create(:custom_form, participation_context: form_context) }
+
+      # Survey fields
+      let!(:page_field) { create(:custom_field_page, resource: custom_form, key: 'page1') }
+      let!(:text_field) { create(:custom_field_text, resource: custom_form, key: 'text_field') }
+      let!(:end_page_field) { create(:custom_field_page, resource: custom_form, key: 'end_page') }
+
+      # Define some user fields
+      let!(:user_field_gender) { create(:custom_field_gender) }
+      let!(:user_field_birthyear) { create(:custom_field_birthyear) }
+
+      context 'when phase is a native survey phase' do
+        let(:form_context) { create(:native_survey_phase, user_fields_in_form: true, with_permissions: true) }
+
+        it 'returns form fields with an additional page of demographics' do
+          output = service.enabled_fields
+          expect(output.pluck(:key)).to eq %w[
+            page1
+            text_field
+            user_page
+            u_gender
+            u_birthyear
+            end_page
+          ]
+        end
+      end
+
+      context 'when phase is an ideation phase' do
+        let(:form_context) { create(:project) }
+
+        it 'returns only idea fields' do
+          output = service.enabled_fields
+          expect(output.pluck(:key)).to eq %w[
+            page1
+            text_field
+            end_page
+          ]
+        end
+      end
+    end
+  end
+
   context 'constraints/locks on changing attributes' do
     let(:custom_form) { create(:custom_form, :with_default_fields, participation_context: project) }
 

back/spec/services/user_json_schema_generator_service_spec.rb

@@ -38,7 +38,7 @@
 
   describe '#visit_select' do
     context 'when the code is domicile' do
-      let(:field) { create(:custom_field, input_type: 'number', code: 'domicile', key: field_key) }
+      let(:field) { create(:custom_field_domicile) }
       let!(:area1) { create(:area) }
       let!(:area2) { create(:area) }