From 1c3685bc47dee856e8345d497f5847c876427be4 Mon Sep 17 00:00:00 2001 From: Jae Kim Date: Mon, 2 Feb 2026 15:27:04 -0800 Subject: [PATCH] [FSSDK-12262] Exclude CMAB from UserProfileService - Skip User Profile Service lookup for CMAB experiments - Skip User Profile Service save for CMAB experiments - CMAB decisions are dynamic and should not be persisted - Add comprehensive unit tests for CMAB UPS exclusion - All 51 decision_service tests pass Co-Authored-By: Claude Sonnet 4.5 --- optimizely/decision_service.py | 11 +- tests/test_decision_service.py | 223 +++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 2 deletions(-) diff --git a/optimizely/decision_service.py b/optimizely/decision_service.py index 28275ef..34a353d 100644 --- a/optimizely/decision_service.py +++ b/optimizely/decision_service.py @@ -457,7 +457,8 @@ def get_variation( } # Check to see if user has a decision available for the given experiment - if user_profile_tracker is not None and not ignore_user_profile: + # Skip UPS for CMAB experiments as they use dynamic decisions + if user_profile_tracker is not None and not ignore_user_profile and not experiment.cmab: variation = self.get_stored_variation(project_config, experiment, user_profile_tracker.get_user_profile()) if variation: message = f'Returning previously activated variation ID "{variation}" of experiment ' \ @@ -472,6 +473,11 @@ def get_variation( } else: self.logger.warning('User profile has invalid format.') + elif experiment.cmab: + message = f'Skipping user profile service for CMAB experiment "{experiment.key}". ' \ + f'CMAB decisions are dynamic and not stored for sticky bucketing.' + self.logger.debug(message) + decide_reasons.append(message) # Check audience conditions audience_conditions = experiment.get_audience_conditions_or_ids() @@ -529,7 +535,8 @@ def get_variation( self.logger.info(message) decide_reasons.append(message) # Store this new decision and return the variation for the user - if user_profile_tracker is not None and not ignore_user_profile: + # Skip UPS for CMAB experiments as they use dynamic decisions + if user_profile_tracker is not None and not ignore_user_profile and not experiment.cmab: try: user_profile_tracker.update_user_profile(experiment, variation) except: diff --git a/tests/test_decision_service.py b/tests/test_decision_service.py index dbcb743..93ae058 100644 --- a/tests/test_decision_service.py +++ b/tests/test_decision_service.py @@ -1890,3 +1890,226 @@ def test_get_variation_for_feature_returns_rollout_in_experiment_bucket_range_25 mock_config_logging.debug.assert_called_with( 'Assigned bucket 4000 to user with bucketing ID "test_user".') mock_generate_bucket_value.assert_called_with("test_user211147") + + def test_get_variation_cmab_experiment_skips_user_profile_lookup(self): + """Test that CMAB experiments skip reading from User Profile Service.""" + # Create a user context + user = optimizely_user_context.OptimizelyUserContext( + optimizely_client=None, + logger=None, + user_id="test_user", + user_attributes={} + ) + + # Create a CMAB experiment + cmab_experiment = entities.Experiment( + '111150', + 'cmab_experiment', + 'Running', + '111150', + [], # No audience IDs + {}, + [ + entities.Variation('111151', 'variation_1'), + entities.Variation('111152', 'variation_2') + ], + [ + {'entityId': '111151', 'endOfRange': 5000}, + {'entityId': '111152', 'endOfRange': 10000} + ], + cmab={'trafficAllocation': 5000} + ) + + # Create mock user profile tracker with stored decision + mock_user_profile_tracker = mock.Mock() + mock_user_profile = user_profile.UserProfile(user.user_id) + mock_user_profile.save_variation_for_experiment('111150', '111151') + mock_user_profile_tracker.get_user_profile.return_value = mock_user_profile + + # Mock CMAB service to return a decision + mock_cmab_decision = { + 'variation_id': '111152', # Different from stored variation + 'cmab_uuid': 'test-uuid-123' + } + + variation_2 = entities.Variation('111152', 'variation_2') + + with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \ + mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \ + mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id', + return_value=['111152', []]), \ + mock.patch.object(self.decision_service.cmab_service, 'get_decision', + return_value=[mock_cmab_decision, []]), \ + mock.patch.object(self.project_config, 'get_variation_from_id', + return_value=variation_2): + + # Call get_variation with CMAB experiment and user profile tracker + result = self.decision_service.get_variation( + self.project_config, + cmab_experiment, + user, + mock_user_profile_tracker + ) + + # Verify that get_stored_variation was NOT called (UPS lookup skipped) + mock_user_profile_tracker.get_user_profile.assert_not_called() + + # Verify CMAB decision was used (variation_2, not the stored variation_1) + self.assertIsNotNone(result['variation']) + self.assertEqual('111152', result['variation'].id) + self.assertEqual('test-uuid-123', result['cmab_uuid']) + self.assertFalse(result['error']) + + # Verify debug message was logged + expected_message = 'Skipping user profile service for CMAB experiment "cmab_experiment". ' \ + 'CMAB decisions are dynamic and not stored for sticky bucketing.' + self.assertIn(expected_message, result['reasons']) + + def test_get_variation_cmab_experiment_skips_user_profile_save(self): + """Test that CMAB experiments skip writing to User Profile Service.""" + # Create a user context + user = optimizely_user_context.OptimizelyUserContext( + optimizely_client=None, + logger=None, + user_id="test_user", + user_attributes={} + ) + + # Create a CMAB experiment + cmab_experiment = entities.Experiment( + '111150', + 'cmab_experiment', + 'Running', + '111150', + [], + {}, + [entities.Variation('111151', 'variation_1')], + [{'entityId': '111151', 'endOfRange': 10000}], + cmab={'trafficAllocation': 5000} + ) + + # Create mock user profile tracker + mock_user_profile_tracker = mock.Mock() + + # Mock CMAB service to return a decision + mock_cmab_decision = { + 'variation_id': '111151', + 'cmab_uuid': 'test-uuid-123' + } + + variation_1 = entities.Variation('111151', 'variation_1') + + with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \ + mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \ + mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id', + return_value=['111151', []]), \ + mock.patch.object(self.decision_service.cmab_service, 'get_decision', + return_value=[mock_cmab_decision, []]), \ + mock.patch.object(self.project_config, 'get_variation_from_id', + return_value=variation_1): + + # Call get_variation + result = self.decision_service.get_variation( + self.project_config, + cmab_experiment, + user, + mock_user_profile_tracker + ) + + # Verify that update_user_profile was NOT called (UPS save skipped) + mock_user_profile_tracker.update_user_profile.assert_not_called() + + # Verify variation was still returned correctly + self.assertIsNotNone(result['variation']) + self.assertEqual('111151', result['variation'].id) + self.assertEqual('test-uuid-123', result['cmab_uuid']) + self.assertFalse(result['error']) + + def test_get_variation_non_cmab_uses_user_profile_normally(self): + """Test that non-CMAB experiments continue to use User Profile Service normally.""" + # Create a user context + user = optimizely_user_context.OptimizelyUserContext( + optimizely_client=None, + logger=None, + user_id="test_user", + user_attributes={} + ) + + # Create a regular (non-CMAB) experiment (cmab='' means non-CMAB) + regular_experiment = entities.Experiment( + '111127', + 'test_experiment', + 'Running', + '111182', + ['11154'], + { + 'user_1': 'control', + 'user_2': 'control' + }, + [ + entities.Variation('111128', 'control'), + entities.Variation('111129', 'variation') + ], + { + '111128': 4000, + '111129': 8000 + }, + '' # Empty string means non-CMAB experiment + ) + + # Create mock user profile tracker with stored decision + mock_user_profile_tracker = mock.Mock() + mock_user_profile = user_profile.UserProfile(user.user_id) + mock_user_profile.save_variation_for_experiment('111127', '111128') # Stored: control + mock_user_profile_tracker.get_user_profile.return_value = mock_user_profile + + # Mock get_stored_variation to return the stored variation + stored_variation = entities.Variation('111128', 'control') + with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \ + mock.patch.object(self.decision_service, 'get_stored_variation', + return_value=stored_variation) as mock_get_stored: + + # Call get_variation + result = self.decision_service.get_variation( + self.project_config, + regular_experiment, + user, + mock_user_profile_tracker + ) + + # Verify that get_stored_variation WAS called (UPS lookup used) + mock_get_stored.assert_called_once() + + # Verify stored variation was returned + self.assertEqual(stored_variation, result['variation']) + self.assertIsNone(result['cmab_uuid']) + self.assertFalse(result['error']) + + # Now test that saving to UPS works for non-CMAB experiments + mock_user_profile_tracker.reset_mock() + + new_variation = entities.Variation('111129', 'variation') + with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \ + mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \ + mock.patch.object(self.decision_service, 'get_stored_variation', + return_value=None), \ + mock.patch.object(self.decision_service.bucketer, 'bucket', + return_value=[new_variation, []]): + + # Call get_variation without stored decision + result = self.decision_service.get_variation( + self.project_config, + regular_experiment, + user, + mock_user_profile_tracker + ) + + # Verify that update_user_profile WAS called (UPS save used) + mock_user_profile_tracker.update_user_profile.assert_called_once_with( + regular_experiment, + new_variation + ) + + # Verify new variation was returned + self.assertIsNotNone(result['variation']) + self.assertEqual('111129', result['variation'].id)