1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // FieldTrial is a class for handling details of statistical experiments 6 // performed by actual users in the field (i.e., in a shipped or beta product). 7 // All code is called exclusively on the UI thread currently. 8 // 9 // The simplest example is an experiment to see whether one of two options 10 // produces "better" results across our user population. In that scenario, UMA 11 // data is uploaded to aggregate the test results, and this FieldTrial class 12 // manages the state of each such experiment (state == which option was 13 // pseudo-randomly selected). 14 // 15 // States are typically generated randomly, either based on a one time 16 // randomization (which will yield the same results, in terms of selecting 17 // the client for a field trial or not, for every run of the program on a 18 // given machine), or by a session randomization (generated each time the 19 // application starts up, but held constant during the duration of the 20 // process). 21 22 //------------------------------------------------------------------------------ 23 // Example: Suppose we have an experiment involving memory, such as determining 24 // the impact of some pruning algorithm. 25 // We assume that we already have a histogram of memory usage, such as: 26 27 // UMA_HISTOGRAM_COUNTS("Memory.RendererTotal", count); 28 29 // Somewhere in main thread initialization code, we'd probably define an 30 // instance of a FieldTrial, with code such as: 31 32 // // FieldTrials are reference counted, and persist automagically until 33 // // process teardown, courtesy of their automatic registration in 34 // // FieldTrialList. 35 // // Note: This field trial will run in Chrome instances compiled through 36 // // 8 July, 2015, and after that all instances will be in "StandardMem". 37 // scoped_refptr<base::FieldTrial> trial( 38 // base::FieldTrialList::FactoryGetFieldTrial( 39 // "MemoryExperiment", 1000, "StandardMem", 2015, 7, 8, 40 // base::FieldTrial::ONE_TIME_RANDOMIZED, NULL)); 41 // 42 // const int high_mem_group = 43 // trial->AppendGroup("HighMem", 20); // 2% in HighMem group. 44 // const int low_mem_group = 45 // trial->AppendGroup("LowMem", 20); // 2% in LowMem group. 46 // // Take action depending of which group we randomly land in. 47 // if (trial->group() == high_mem_group) 48 // SetPruningAlgorithm(kType1); // Sample setting of browser state. 49 // else if (trial->group() == low_mem_group) 50 // SetPruningAlgorithm(kType2); // Sample alternate setting. 51 52 //------------------------------------------------------------------------------ 53 54 #ifndef BASE_METRICS_FIELD_TRIAL_H_ 55 #define BASE_METRICS_FIELD_TRIAL_H_ 56 57 #include <stddef.h> 58 #include <stdint.h> 59 60 #include <map> 61 #include <set> 62 #include <string> 63 #include <vector> 64 65 #include "base/base_export.h" 66 #include "base/gtest_prod_util.h" 67 #include "base/macros.h" 68 #include "base/memory/ref_counted.h" 69 #include "base/observer_list_threadsafe.h" 70 #include "base/strings/string_piece.h" 71 #include "base/synchronization/lock.h" 72 #include "base/time/time.h" 73 74 namespace base { 75 76 class FieldTrialList; 77 78 class BASE_EXPORT FieldTrial : public RefCounted<FieldTrial> { 79 public: 80 typedef int Probability; // Probability type for being selected in a trial. 81 82 // Specifies the persistence of the field trial group choice. 83 enum RandomizationType { 84 // One time randomized trials will persist the group choice between 85 // restarts, which is recommended for most trials, especially those that 86 // change user visible behavior. 87 ONE_TIME_RANDOMIZED, 88 // Session randomized trials will roll the dice to select a group on every 89 // process restart. 90 SESSION_RANDOMIZED, 91 }; 92 93 // EntropyProvider is an interface for providing entropy for one-time 94 // randomized (persistent) field trials. 95 class BASE_EXPORT EntropyProvider { 96 public: 97 virtual ~EntropyProvider(); 98 99 // Returns a double in the range of [0, 1) to be used for the dice roll for 100 // the specified field trial. If |randomization_seed| is not 0, it will be 101 // used in preference to |trial_name| for generating the entropy by entropy 102 // providers that support it. A given instance should always return the same 103 // value given the same input |trial_name| and |randomization_seed| values. 104 virtual double GetEntropyForTrial(const std::string& trial_name, 105 uint32_t randomization_seed) const = 0; 106 }; 107 108 // A pair representing a Field Trial and its selected group. 109 struct ActiveGroup { 110 std::string trial_name; 111 std::string group_name; 112 }; 113 114 // A triplet representing a FieldTrial, its selected group and whether it's 115 // active. 116 struct BASE_EXPORT State { 117 StringPiece trial_name; 118 StringPiece group_name; 119 bool activated; 120 121 State(); 122 ~State(); 123 }; 124 125 typedef std::vector<ActiveGroup> ActiveGroups; 126 127 // A return value to indicate that a given instance has not yet had a group 128 // assignment (and hence is not yet participating in the trial). 129 static const int kNotFinalized; 130 131 // Disables this trial, meaning it always determines the default group 132 // has been selected. May be called immediately after construction, or 133 // at any time after initialization (should not be interleaved with 134 // AppendGroup calls). Once disabled, there is no way to re-enable a 135 // trial. 136 // TODO(mad): http://code.google.com/p/chromium/issues/detail?id=121446 137 // This doesn't properly reset to Default when a group was forced. 138 void Disable(); 139 140 // Establish the name and probability of the next group in this trial. 141 // Sometimes, based on construction randomization, this call may cause the 142 // provided group to be *THE* group selected for use in this instance. 143 // The return value is the group number of the new group. 144 int AppendGroup(const std::string& name, Probability group_probability); 145 146 // Return the name of the FieldTrial (excluding the group name). 147 const std::string& trial_name() const { return trial_name_; } 148 149 // Return the randomly selected group number that was assigned, and notify 150 // any/all observers that this finalized group number has presumably been used 151 // (queried), and will never change. Note that this will force an instance to 152 // participate, and make it illegal to attempt to probabilistically add any 153 // other groups to the trial. 154 int group(); 155 156 // If the group's name is empty, a string version containing the group number 157 // is used as the group name. This causes a winner to be chosen if none was. 158 const std::string& group_name(); 159 160 // Finalizes the group choice and returns the chosen group, but does not mark 161 // the trial as active - so its state will not be reported until group_name() 162 // or similar is called. 163 const std::string& GetGroupNameWithoutActivation(); 164 165 // Set the field trial as forced, meaning that it was setup earlier than 166 // the hard coded registration of the field trial to override it. 167 // This allows the code that was hard coded to register the field trial to 168 // still succeed even though the field trial has already been registered. 169 // This must be called after appending all the groups, since we will make 170 // the group choice here. Note that this is a NOOP for already forced trials. 171 // And, as the rest of the FieldTrial code, this is not thread safe and must 172 // be done from the UI thread. 173 void SetForced(); 174 175 // Enable benchmarking sets field trials to a common setting. 176 static void EnableBenchmarking(); 177 178 // Creates a FieldTrial object with the specified parameters, to be used for 179 // simulation of group assignment without actually affecting global field 180 // trial state in the running process. Group assignment will be done based on 181 // |entropy_value|, which must have a range of [0, 1). 182 // 183 // Note: Using this function will not register the field trial globally in the 184 // running process - for that, use FieldTrialList::FactoryGetFieldTrial(). 185 // 186 // The ownership of the returned FieldTrial is transfered to the caller which 187 // is responsible for deref'ing it (e.g. by using scoped_refptr<FieldTrial>). 188 static FieldTrial* CreateSimulatedFieldTrial( 189 const std::string& trial_name, 190 Probability total_probability, 191 const std::string& default_group_name, 192 double entropy_value); 193 194 private: 195 // Allow tests to access our innards for testing purposes. 196 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Registration); 197 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, AbsoluteProbabilities); 198 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, RemainingProbability); 199 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, FiftyFiftyProbability); 200 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, MiddleProbabilities); 201 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, OneWinner); 202 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DisableProbability); 203 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroups); 204 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, AllGroups); 205 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroupsNotFinalized); 206 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Save); 207 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SaveAll); 208 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DuplicateRestore); 209 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOff); 210 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOn); 211 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_Default); 212 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_NonDefault); 213 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, FloatBoundariesGiveEqualGroupSizes); 214 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DoesNotSurpassTotalProbability); 215 216 friend class base::FieldTrialList; 217 218 friend class RefCounted<FieldTrial>; 219 220 // This is the group number of the 'default' group when a choice wasn't forced 221 // by a call to FieldTrialList::CreateFieldTrial. It is kept private so that 222 // consumers don't use it by mistake in cases where the group was forced. 223 static const int kDefaultGroupNumber; 224 225 // Creates a field trial with the specified parameters. Group assignment will 226 // be done based on |entropy_value|, which must have a range of [0, 1). 227 FieldTrial(const std::string& trial_name, 228 Probability total_probability, 229 const std::string& default_group_name, 230 double entropy_value); 231 virtual ~FieldTrial(); 232 233 // Return the default group name of the FieldTrial. 234 std::string default_group_name() const { return default_group_name_; } 235 236 // Marks this trial as having been registered with the FieldTrialList. Must be 237 // called no more than once and before any |group()| calls have occurred. 238 void SetTrialRegistered(); 239 240 // Sets the chosen group name and number. 241 void SetGroupChoice(const std::string& group_name, int number); 242 243 // Ensures that a group is chosen, if it hasn't yet been. The field trial 244 // might yet be disabled, so this call will *not* notify observers of the 245 // status. 246 void FinalizeGroupChoice(); 247 248 // Returns the trial name and selected group name for this field trial via 249 // the output parameter |active_group|, but only if the group has already 250 // been chosen and has been externally observed via |group()| and the trial 251 // has not been disabled. In that case, true is returned and |active_group| 252 // is filled in; otherwise, the result is false and |active_group| is left 253 // untouched. 254 bool GetActiveGroup(ActiveGroup* active_group) const; 255 256 // Returns the trial name and selected group name for this field trial via 257 // the output parameter |field_trial_state|, but only if the trial has not 258 // been disabled. In that case, true is returned and |field_trial_state| is 259 // filled in; otherwise, the result is false and |field_trial_state| is left 260 // untouched. 261 bool GetState(State* field_trial_state); 262 263 // Returns the group_name. A winner need not have been chosen. 264 std::string group_name_internal() const { return group_name_; } 265 266 // The name of the field trial, as can be found via the FieldTrialList. 267 const std::string trial_name_; 268 269 // The maximum sum of all probabilities supplied, which corresponds to 100%. 270 // This is the scaling factor used to adjust supplied probabilities. 271 const Probability divisor_; 272 273 // The name of the default group. 274 const std::string default_group_name_; 275 276 // The randomly selected probability that is used to select a group (or have 277 // the instance not participate). It is the product of divisor_ and a random 278 // number between [0, 1). 279 Probability random_; 280 281 // Sum of the probabilities of all appended groups. 282 Probability accumulated_group_probability_; 283 284 // The number that will be returned by the next AppendGroup() call. 285 int next_group_number_; 286 287 // The pseudo-randomly assigned group number. 288 // This is kNotFinalized if no group has been assigned. 289 int group_; 290 291 // A textual name for the randomly selected group. Valid after |group()| 292 // has been called. 293 std::string group_name_; 294 295 // When enable_field_trial_ is false, field trial reverts to the 'default' 296 // group. 297 bool enable_field_trial_; 298 299 // When forced_ is true, we return the chosen group from AppendGroup when 300 // appropriate. 301 bool forced_; 302 303 // Specifies whether the group choice has been reported to observers. 304 bool group_reported_; 305 306 // Whether this trial is registered with the global FieldTrialList and thus 307 // should notify it when its group is queried. 308 bool trial_registered_; 309 310 // When benchmarking is enabled, field trials all revert to the 'default' 311 // group. 312 static bool enable_benchmarking_; 313 314 DISALLOW_COPY_AND_ASSIGN(FieldTrial); 315 }; 316 317 //------------------------------------------------------------------------------ 318 // Class with a list of all active field trials. A trial is active if it has 319 // been registered, which includes evaluating its state based on its probaility. 320 // Only one instance of this class exists. 321 class BASE_EXPORT FieldTrialList { 322 public: 323 // Specifies whether field trials should be activated (marked as "used"), when 324 // created using |CreateTrialsFromString()|. Has no effect on trials that are 325 // prefixed with |kActivationMarker|, which will always be activated." 326 enum FieldTrialActivationMode { 327 DONT_ACTIVATE_TRIALS, 328 ACTIVATE_TRIALS, 329 }; 330 331 // Year that is guaranteed to not be expired when instantiating a field trial 332 // via |FactoryGetFieldTrial()|. Set to two years from the build date. 333 static int kNoExpirationYear; 334 335 // Observer is notified when a FieldTrial's group is selected. 336 class BASE_EXPORT Observer { 337 public: 338 // Notify observers when FieldTrials's group is selected. 339 virtual void OnFieldTrialGroupFinalized(const std::string& trial_name, 340 const std::string& group_name) = 0; 341 342 protected: 343 virtual ~Observer(); 344 }; 345 346 // This singleton holds the global list of registered FieldTrials. 347 // 348 // To support one-time randomized field trials, specify a non-NULL 349 // |entropy_provider| which should be a source of uniformly distributed 350 // entropy values. Takes ownership of |entropy_provider|. If one time 351 // randomization is not desired, pass in NULL for |entropy_provider|. 352 explicit FieldTrialList(const FieldTrial::EntropyProvider* entropy_provider); 353 354 // Destructor Release()'s references to all registered FieldTrial instances. 355 ~FieldTrialList(); 356 357 // Get a FieldTrial instance from the factory. 358 // 359 // |name| is used to register the instance with the FieldTrialList class, 360 // and can be used to find the trial (only one trial can be present for each 361 // name). |default_group_name| is the name of the default group which will 362 // be chosen if none of the subsequent appended groups get to be chosen. 363 // |default_group_number| can receive the group number of the default group as 364 // AppendGroup returns the number of the subsequence groups. |trial_name| and 365 // |default_group_name| may not be empty but |default_group_number| can be 366 // NULL if the value is not needed. 367 // 368 // Group probabilities that are later supplied must sum to less than or equal 369 // to the |total_probability|. Arguments |year|, |month| and |day_of_month| 370 // specify the expiration time. If the build time is after the expiration time 371 // then the field trial reverts to the 'default' group. 372 // 373 // Use this static method to get a startup-randomized FieldTrial or a 374 // previously created forced FieldTrial. 375 static FieldTrial* FactoryGetFieldTrial( 376 const std::string& trial_name, 377 FieldTrial::Probability total_probability, 378 const std::string& default_group_name, 379 const int year, 380 const int month, 381 const int day_of_month, 382 FieldTrial::RandomizationType randomization_type, 383 int* default_group_number); 384 385 // Same as FactoryGetFieldTrial(), but allows specifying a custom seed to be 386 // used on one-time randomized field trials (instead of a hash of the trial 387 // name, which is used otherwise or if |randomization_seed| has value 0). The 388 // |randomization_seed| value (other than 0) should never be the same for two 389 // trials, else this would result in correlated group assignments. 390 // Note: Using a custom randomization seed is only supported by the 391 // PermutedEntropyProvider (which is used when UMA is not enabled). 392 static FieldTrial* FactoryGetFieldTrialWithRandomizationSeed( 393 const std::string& trial_name, 394 FieldTrial::Probability total_probability, 395 const std::string& default_group_name, 396 const int year, 397 const int month, 398 const int day_of_month, 399 FieldTrial::RandomizationType randomization_type, 400 uint32_t randomization_seed, 401 int* default_group_number); 402 403 // The Find() method can be used to test to see if a named trial was already 404 // registered, or to retrieve a pointer to it from the global map. 405 static FieldTrial* Find(const std::string& trial_name); 406 407 // Returns the group number chosen for the named trial, or 408 // FieldTrial::kNotFinalized if the trial does not exist. 409 static int FindValue(const std::string& trial_name); 410 411 // Returns the group name chosen for the named trial, or the empty string if 412 // the trial does not exist. The first call of this function on a given field 413 // trial will mark it as active, so that its state will be reported with usage 414 // metrics, crashes, etc. 415 static std::string FindFullName(const std::string& trial_name); 416 417 // Returns true if the named trial has been registered. 418 static bool TrialExists(const std::string& trial_name); 419 420 // Returns true if the named trial exists and has been activated. 421 static bool IsTrialActive(const std::string& trial_name); 422 423 // Creates a persistent representation of active FieldTrial instances for 424 // resurrection in another process. This allows randomization to be done in 425 // one process, and secondary processes can be synchronized on the result. 426 // The resulting string contains the name and group name pairs of all 427 // registered FieldTrials for which the group has been chosen and externally 428 // observed (via |group()|) and which have not been disabled, with "/" used 429 // to separate all names and to terminate the string. This string is parsed 430 // by |CreateTrialsFromString()|. 431 static void StatesToString(std::string* output); 432 433 // Creates a persistent representation of all FieldTrial instances for 434 // resurrection in another process. This allows randomization to be done in 435 // one process, and secondary processes can be synchronized on the result. 436 // The resulting string contains the name and group name pairs of all 437 // registered FieldTrials which have not been disabled, with "/" used 438 // to separate all names and to terminate the string. All activated trials 439 // have their name prefixed with "*". This string is parsed by 440 // |CreateTrialsFromString()|. 441 static void AllStatesToString(std::string* output); 442 443 // Fills in the supplied vector |active_groups| (which must be empty when 444 // called) with a snapshot of all registered FieldTrials for which the group 445 // has been chosen and externally observed (via |group()|) and which have 446 // not been disabled. 447 static void GetActiveFieldTrialGroups( 448 FieldTrial::ActiveGroups* active_groups); 449 450 // Returns the field trials that are marked active in |trials_string|. 451 static void GetActiveFieldTrialGroupsFromString( 452 const std::string& trials_string, 453 FieldTrial::ActiveGroups* active_groups); 454 455 // Use a state string (re: StatesToString()) to augment the current list of 456 // field trials to include the supplied trials, and using a 100% probability 457 // for each trial, force them to have the same group string. This is commonly 458 // used in a non-browser process, to carry randomly selected state in a 459 // browser process into this non-browser process, but could also be invoked 460 // through a command line argument to the browser process. The created field 461 // trials are all marked as "used" for the purposes of active trial reporting 462 // if |mode| is ACTIVATE_TRIALS, otherwise each trial will be marked as "used" 463 // if it is prefixed with |kActivationMarker|. Trial names in 464 // |ignored_trial_names| are ignored when parsing |trials_string|. 465 static bool CreateTrialsFromString( 466 const std::string& trials_string, 467 FieldTrialActivationMode mode, 468 const std::set<std::string>& ignored_trial_names); 469 470 // Create a FieldTrial with the given |name| and using 100% probability for 471 // the FieldTrial, force FieldTrial to have the same group string as 472 // |group_name|. This is commonly used in a non-browser process, to carry 473 // randomly selected state in a browser process into this non-browser process. 474 // It returns NULL if there is a FieldTrial that is already registered with 475 // the same |name| but has different finalized group string (|group_name|). 476 static FieldTrial* CreateFieldTrial(const std::string& name, 477 const std::string& group_name); 478 479 // Add an observer to be notified when a field trial is irrevocably committed 480 // to being part of some specific field_group (and hence the group_name is 481 // also finalized for that field_trial). 482 static void AddObserver(Observer* observer); 483 484 // Remove an observer. 485 static void RemoveObserver(Observer* observer); 486 487 // Notify all observers that a group has been finalized for |field_trial|. 488 static void NotifyFieldTrialGroupSelection(FieldTrial* field_trial); 489 490 // Return the number of active field trials. 491 static size_t GetFieldTrialCount(); 492 493 private: 494 // A map from FieldTrial names to the actual instances. 495 typedef std::map<std::string, FieldTrial*> RegistrationMap; 496 497 // If one-time randomization is enabled, returns a weak pointer to the 498 // corresponding EntropyProvider. Otherwise, returns NULL. 499 static const FieldTrial::EntropyProvider* 500 GetEntropyProviderForOneTimeRandomization(); 501 502 // Helper function should be called only while holding lock_. 503 FieldTrial* PreLockedFind(const std::string& name); 504 505 // Register() stores a pointer to the given trial in a global map. 506 // This method also AddRef's the indicated trial. 507 // This should always be called after creating a new FieldTrial instance. 508 static void Register(FieldTrial* trial); 509 510 static FieldTrialList* global_; // The singleton of this class. 511 512 // This will tell us if there is an attempt to register a field 513 // trial or check if one-time randomization is enabled without 514 // creating the FieldTrialList. This is not an error, unless a 515 // FieldTrialList is created after that. 516 static bool used_without_global_; 517 518 // Lock for access to registered_. 519 base::Lock lock_; 520 RegistrationMap registered_; 521 522 // Entropy provider to be used for one-time randomized field trials. If NULL, 523 // one-time randomization is not supported. 524 scoped_ptr<const FieldTrial::EntropyProvider> entropy_provider_; 525 526 // List of observers to be notified when a group is selected for a FieldTrial. 527 scoped_refptr<ObserverListThreadSafe<Observer> > observer_list_; 528 529 DISALLOW_COPY_AND_ASSIGN(FieldTrialList); 530 }; 531 532 } // namespace base 533 534 #endif // BASE_METRICS_FIELD_TRIAL_H_ 535