my_exp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. experiment design"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.0 function and parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['--l_rate'], dest='l_rate', nargs=None, const=None, default=0.001, type=<class 'float'>, choices=None, required=False, help='learning rage', metavar=None)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import argparse\n",
    "import math\n",
    "from scipy import stats\n",
    "import os\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "#let PID system adaptively changes bids.\n",
    "def PID_sys(current_cpc_, target_cpc_, current_ecpc_):\n",
    "    #Here we use a simplified version of PID: (1) k_I=p_D=0, (2) set current_ecpc = current_cpc\n",
    "    #which means this simplified PID use an exponential smoothing to update bids\n",
    "    \n",
    "    k_p = 0.4\n",
    "    \n",
    "    #equivalent version:\n",
    "    #error = target_cpc_ - current_ecpc_\n",
    "    #current_cpc_updated = current_cpc_ + k_p*error\n",
    "    current_cpc_updated = k_p * target_cpc_ + (1-k_p) * current_cpc_\n",
    "    return current_cpc_updated\n",
    "\n",
    "\n",
    "#Define context generating function.\n",
    "gender_pdf = np.array([0.5, 0.5])#marginal distribution of gender. [male, female]\n",
    "location_pdf = np.array([0.22, 0.46, 0.32])#marginal distribution of city level. [large city, medium city, small city]\n",
    "age_seg_pdf = np.array([0.46, 0.34, 0.20])#marginal distribution of age. [small age, middle age, large age]\n",
    "dim_context  = int(np.shape(gender_pdf)[0]+np.shape(location_pdf)[0]+np.shape(age_seg_pdf)[0] + 1)\n",
    "def context_generate(gender_ratio = gender_pdf, loc_ratio = location_pdf, age_ratio = age_seg_pdf):\n",
    "    a = np.random.multinomial(1, gender_pdf, size=1)\n",
    "    b = np.random.multinomial(1, location_pdf, size=1)\n",
    "    c = np.random.multinomial(1, age_ratio, size=1)\n",
    "    return np.concatenate((a, b, c, 1), axis=None)\n",
    "\n",
    "\n",
    "#Load sampled data and define ad-campaign generating function.\n",
    "#load joint information of ctr fixed effect coefficient, and target bid (CPC)\n",
    "ctr_bid_np = np.load('ctr_bid_data.npy')\n",
    "#note the first 200 rows are marure ads and last 100 rows are new ads\n",
    "def generate_ad(n, new_raito = 0.30, dim_context = dim_context, data = ctr_bid_np):\n",
    "    #n is the total number of ads, new_raito is ratio of new ads\n",
    "    num_new_ad = int(n*new_raito)\n",
    "    num_mature_ad = n - num_new_ad\n",
    "    sampled_ctr = data[:, 0]\n",
    "    sampled_cpc = data[:, 1]\n",
    "    \n",
    "    #initialize output data: \n",
    "    is_new_index = np.zeros(n)\n",
    "    target_cpc = np.zeros(n)\n",
    "    coefficient_ctr_model = np.zeros([n, dim_context])\n",
    "    \n",
    "    #generate bid, true CTR model\n",
    "    for i in range(n):\n",
    "        index = 0\n",
    "        if i < num_mature_ad:\n",
    "            is_new_index[i] = 0\n",
    "            index = np.random.choice(np.arange(0,200))\n",
    "        else:\n",
    "            is_new_index[i] = 1\n",
    "            index = np.random.choice(np.arange(200,300)) \n",
    "        target_cpc[i] = sampled_cpc[index]\n",
    "        coefficient_ctr_model[i, 0:-1] = np.random.uniform(-0.05,0.05,dim_context-1)\n",
    "        coefficient_ctr_model[i, -1] = sampled_ctr[index]\n",
    "    return np.int32(is_new_index), target_cpc, coefficient_ctr_model\n",
    "\n",
    "# hyperparameters settings. Notice that this system is based on first price auction and CPC,\n",
    "import sys\n",
    "sys.argv=['']\n",
    "del sys\n",
    "parser = argparse.ArgumentParser()\n",
    "\n",
    "#Default cold start parameters - the cold start value = reward*I(clicks>=target_click)\n",
    "parser.add_argument('--total_ads', type=int, default=1000, help='total number of ads for simulation')\n",
    "parser.add_argument('--new_ad_ratio', type=float, default=0.3, help='ratio of new ads')\n",
    "parser.add_argument('--total_pv', type=int, default=1000000, help='loaded total page views in one cold start horizon for simulation')\n",
    "parser.add_argument('--reward', type=float, default=1000000, help='reward of cold start')\n",
    "parser.add_argument('--delta',  type=float, default=200000, help='delta reward of treated group')\n",
    "parser.add_argument('--target_click', type=int, default=1000, help='target click')\n",
    "parser.add_argument('--alpha', type=float, default=1e-3, help='target click/total_pv')\n",
    "#If one wants to change total_ads, it is better to proportionally change total_pv to balance the two-side market\n",
    "\n",
    "#Others parameters\n",
    "parser.add_argument('--budget_limit', type=int, default=5e5, help='set an uniform budget limit on ads')\n",
    "parser.add_argument('--stepsize', type=float, default=30, help='step size of DMD')\n",
    "parser.add_argument('--PID_cycle', type=int, default=100000, help='after every # pvs, update PID')\n",
    "\n",
    "#Parameters about experiments - sampling rate\n",
    "parser.add_argument('--ad_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "parser.add_argument('--pv_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "\n",
    "#Parameters about model - learning rate\n",
    "parser.add_argument('--l_rate', type=float, default=0.001, help='learning rage')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.1 all-control"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_control(opt, is_new_index, target_bid_price, coefficient_ctr_model):\n",
    "    \n",
    "    #Initialize the simulation\n",
    "    target_cpc = np.copy(target_bid_price)\n",
    "    total_clicks = np.zeros(opt.total_ads)#accumulated number of clicks\n",
    "    total_impressions = np.zeros(opt.total_ads)#accumulated number of impressions\n",
    "    total_cost = np.zeros(opt.total_ads)#accumulated cost of ads\n",
    "    coefficient_ctr_pred_model = np.zeros([opt.total_ads, dim_context]) \n",
    "\n",
    "    #assume for mature ads, we know exactly the true CTR model\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==0:\n",
    "            coefficient_ctr_pred_model[i] = coefficient_ctr_model[i]\n",
    "        else:\n",
    "            #for new ads, initialize average of real CTR and mean CTR = 0.067 (over all new ads)\n",
    "            #to mimic real situation, reco sys has partial prior knowledge of new ads\n",
    "            #typically initialize by avergae over all ads with the same industry\n",
    "            coefficient_ctr_pred_model[i, -1] = 0.5*coefficient_ctr_model[i, -1] + 0.5*0.067\n",
    "    new_ad_index = np.where(is_new_index==1)[0]\n",
    "\n",
    "    #Initialize PID system\n",
    "    current_cpc = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc[i] = target_cpc[i] + opt.reward/opt.target_click\n",
    "\n",
    "    #simulation starts\n",
    "    for i in tqdm(range(opt.total_pv), desc=\"All control experiment\"):\n",
    "\n",
    "        ##########################generate context##########################\n",
    "        contexts = context_generate()\n",
    "\n",
    "        ##########################calculate eCPM and Auction##########################\n",
    "        #calculate pCTR and eCPM = pCTR*bid\n",
    "        current_bid_np = np.zeros(opt.total_ads)\n",
    "        pCTR = np.maximum(0, np.minimum(np.matmul(coefficient_ctr_pred_model, contexts), 1))\n",
    "\n",
    "        current_bid_np = target_cpc*(1-is_new_index) + current_cpc*is_new_index#if using PID\n",
    "        current_bid_np -= 10000*np.greater(total_cost, opt.budget_limit*np.ones(opt.total_ads))\n",
    "        ecpm = pCTR * current_bid_np\n",
    "        #ranking\n",
    "        max_index = np.argmax(ecpm, axis=0)\n",
    "\n",
    "        ##########################click feedback and cost reduction##########################  \n",
    "        total_impressions[max_index] += 1\n",
    "        #get the feedback\n",
    "        real_ctr = np.matmul(coefficient_ctr_model[max_index], contexts)\n",
    "        is_clicked = np.random.binomial(1, max(0, min(real_ctr, 1)))\n",
    "        if is_clicked == 1:#if clicked\n",
    "            total_clicks[max_index] += 1\n",
    "            total_cost[max_index] += target_cpc[max_index]\n",
    "\n",
    "        ##########################update pCTR prediction model##########################\n",
    "        if is_new_index[max_index]>0:#only update ctr model for new ads since we assume mature ad prediction is accurate\n",
    "            coefficient_ctr_pred_model[max_index] += opt.l_rate*(is_clicked-pCTR[max_index])*contexts#SGD for linear models\n",
    "\n",
    "        ##########################update PID##########################\n",
    "        #using PID system to update current bid\n",
    "        if i%opt.PID_cycle==(opt.PID_cycle-1):\n",
    "            current_ecpc = total_cost/(total_clicks+0.01)#compute ecpc, +0.01 to avoid 0/0=NaN\n",
    "            current_cpc = PID_sys(current_cpc, target_cpc, current_ecpc)\n",
    "        current_cpc = np.less(total_clicks, opt.target_click)*current_cpc + (1- np.less(total_clicks, opt.target_click))*target_cpc\n",
    "    \n",
    "    \n",
    "    #Output simulation results\n",
    "    print('================================================')\n",
    "    rev_new_avg = np.sum(total_cost*is_new_index)/( np.sum(is_new_index)*opt.total_pv)\n",
    "    rev_new_tot = np.sum(total_cost*is_new_index)\n",
    "    print('Average Revenue in new:', rev_new_avg)\n",
    "    print('Total   Revenue in new:', rev_new_tot)\n",
    "    \n",
    "    clicks_new_avg = np.sum(total_clicks*is_new_index)/( np.sum(is_new_index)*opt.total_pv)\n",
    "    clicks_new_tot = np.sum(total_clicks*is_new_index)\n",
    "    print('Average clicks in new:', clicks_new_avg)\n",
    "    print('Total   clicks in new:', clicks_new_tot)\n",
    "    \n",
    "    coldstart_rate = (total_clicks*is_new_index>=opt.target_click).sum()/is_new_index.sum()#potential outcome in control, take Cold Start Success Rate as example\n",
    "    print('Cold Start Success Rate:', coldstart_rate)\n",
    "    print('================================================')\n",
    "    \n",
    "    return (rev_new_avg, rev_new_tot, \n",
    "            clicks_new_avg, clicks_new_tot, \n",
    "            coldstart_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.2 all-treat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_treat(opt, is_new_index, target_bid_price, coefficient_ctr_model):\n",
    "    #Initialize the simulation\n",
    "    target_cpc = np.copy(target_bid_price)\n",
    "    total_clicks = np.zeros(opt.total_ads)#accumulated number of clicks\n",
    "    total_impressions = np.zeros(opt.total_ads)#accumulated number of impressions\n",
    "    total_cost = np.zeros(opt.total_ads)#accumulated cost of ads\n",
    "    coefficient_ctr_pred_model = np.zeros([opt.total_ads, dim_context]) \n",
    "\n",
    "    #assume for mature ads, we know exactly the true CTR model\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==0:\n",
    "            coefficient_ctr_pred_model[i] = coefficient_ctr_model[i]\n",
    "        else:\n",
    "            #for new ads, initialize average of real CTR and mean CTR = 0.067 (over all new ads)\n",
    "            #to mimic real situation, reco sys has partial prior knowledge of new ads\n",
    "            #typically initialize by avergae over all ads with the same industry\n",
    "            coefficient_ctr_pred_model[i, -1] = 0.5*coefficient_ctr_model[i, -1] + 0.5*0.067\n",
    "    new_ad_index = np.where(is_new_index==1)[0]\n",
    "\n",
    "    #Initialize PID system\n",
    "    current_cpc = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc[i] = target_cpc[i] + (opt.reward+opt.delta)/opt.target_click\n",
    "\n",
    "    #simulation starts\n",
    "    for i in tqdm(range(opt.total_pv), desc=\"All treat experiment\"):\n",
    "\n",
    "        ##########################generate context##########################\n",
    "        contexts = context_generate()\n",
    "\n",
    "        ##########################calculate eCPM and Auction##########################\n",
    "        #calculate pCTR and eCPM = pCTR*bid\n",
    "        current_bid_np = np.zeros(opt.total_ads)\n",
    "        pCTR = np.maximum(0, np.minimum(np.matmul(coefficient_ctr_pred_model, contexts), 1))\n",
    "\n",
    "        current_bid_np = target_cpc*(1-is_new_index) + current_cpc*is_new_index#if using PID\n",
    "        current_bid_np -= 10000*np.greater(total_cost, opt.budget_limit*np.ones(opt.total_ads))\n",
    "        ecpm = pCTR * current_bid_np\n",
    "        #ranking\n",
    "        max_index = np.argmax(ecpm, axis=0)\n",
    "\n",
    "        ##########################click feedback and cost reduction##########################  \n",
    "        total_impressions[max_index] += 1\n",
    "        #get the feedback\n",
    "        real_ctr = np.matmul(coefficient_ctr_model[max_index], contexts)\n",
    "        is_clicked = np.random.binomial(1, max(0, min(real_ctr, 1)))\n",
    "        if is_clicked == 1:#if clicked\n",
    "            total_clicks[max_index] += 1\n",
    "            total_cost[max_index] += target_cpc[max_index]\n",
    "\n",
    "        ##########################update pCTR prediction model##########################\n",
    "        if is_new_index[max_index]>0:#only update ctr model for new ads since we assume mature ad prediction is accurate\n",
    "            coefficient_ctr_pred_model[max_index] += opt.l_rate*(is_clicked-pCTR[max_index])*contexts#SGD for linear models\n",
    "\n",
    "\n",
    "        ##########################update PID##########################\n",
    "        #using PID system to update current bid\n",
    "        if i%opt.PID_cycle==(opt.PID_cycle-1):\n",
    "            current_ecpc = total_cost/(total_clicks+0.01)#compute ecpc, +0.01 to avoid 0/0=NaN\n",
    "            current_cpc = PID_sys(current_cpc, target_cpc, current_ecpc)\n",
    "        current_cpc = np.less(total_clicks, opt.target_click)*current_cpc + (1- np.less(total_clicks, opt.target_click))*target_cpc\n",
    "        \n",
    "    #Output simulation results\n",
    "    print('================================================')\n",
    "    rev_new_avg = np.sum(total_cost*is_new_index)/( np.sum(is_new_index)*opt.total_pv)\n",
    "    rev_new_tot = np.sum(total_cost*is_new_index)\n",
    "    print('Average Revenue in new:', rev_new_avg)\n",
    "    print('Total   Revenue in new:', rev_new_tot)\n",
    "    \n",
    "    clicks_new_avg = np.sum(total_clicks*is_new_index)/( np.sum(is_new_index)*opt.total_pv)\n",
    "    clicks_new_tot = np.sum(total_clicks*is_new_index)\n",
    "    print('Average clicks in new:', clicks_new_avg)\n",
    "    print('Total   clicks in new:', clicks_new_tot)\n",
    "    \n",
    "    coldstart_rate = (total_clicks*is_new_index>=opt.target_click).sum()/is_new_index.sum()#potential outcome in control, take Cold Start Success Rate as example\n",
    "    print('Cold Start Success Rate:', coldstart_rate)\n",
    "    print('================================================')\n",
    "    return (rev_new_avg, rev_new_tot, \n",
    "            clicks_new_avg, clicks_new_tot, \n",
    "            coldstart_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.3 ad-side exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ad_exp(opt, is_new_index, target_bid_price, coefficient_ctr_model):\n",
    "    print('Ad-side experiment')\n",
    "    print('# of new ads in treatment condition', int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "    print('# of new ads in control condition', int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "\n",
    "    #randomly assign new ads into treatment and control groups\n",
    "    is_new_treatment = np.zeros(opt.total_ads)\n",
    "    is_new_control = np.zeros(opt.total_ads)\n",
    "\n",
    "    new_ad_index = np.where(is_new_index==1)[0]\n",
    "    treatment_new_ad_index = np.random.choice(new_ad_index, int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio), replace=False)\n",
    "\n",
    "    control_new_ad_index = np.random.choice(np.setdiff1d(new_ad_index, treatment_new_ad_index), \n",
    "                                              int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio), replace=False)\n",
    "\n",
    "    for i in range(int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio)):\n",
    "        is_new_treatment[treatment_new_ad_index[i]] = 1\n",
    "        is_new_control[control_new_ad_index[i]] = 1\n",
    "\n",
    "    #initialization\n",
    "    target_cpc = np.copy(target_bid_price)\n",
    "    total_clicks = np.zeros(opt.total_ads)#accumulated number of clicks\n",
    "    total_impressions = np.zeros(opt.total_ads)#accumulated number of impressions\n",
    "    total_cost = np.zeros(opt.total_ads)#accumulated cost of ads\n",
    "    coefficient_ctr_pred_model = np.zeros([opt.total_ads, dim_context]) \n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==0:\n",
    "            coefficient_ctr_pred_model[i] = coefficient_ctr_model[i]\n",
    "        else:\n",
    "            coefficient_ctr_pred_model[i, -1] = 0.5*coefficient_ctr_model[i, -1]+0.5*0.067\n",
    "    current_cpc = np.zeros(opt.total_ads)\n",
    "\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_control[i] == 1:\n",
    "            #new control ads\n",
    "            current_cpc[i] = target_cpc[i] + opt.reward/opt.target_click\n",
    "        if is_new_treatment[i] == 1:\n",
    "            #new treatment ads\n",
    "            current_cpc[i] = target_cpc[i] + (opt.reward + opt.delta) /opt.target_click\n",
    "\n",
    "        if is_new_index[i] == 1 and is_new_control[i] != 1 and is_new_treatment[i] != 1:\n",
    "            #other control ads\n",
    "            current_cpc[i] = target_cpc[i] + opt.reward/opt.target_click\n",
    "\n",
    "    #simulation starts\n",
    "    for i in tqdm(range(opt.total_pv), desc=\"ad side experiment\"):\n",
    "\n",
    "        ##########################generate context##########################\n",
    "        contexts = context_generate()\n",
    "\n",
    "        ##########################calculate eCPM and Auction##########################\n",
    "        #calculate pCTR and eCPM = pCTR*bid\n",
    "        current_bid_np = np.copy(target_cpc)\n",
    "        pCTR = np.maximum(0, np.minimum(np.matmul(coefficient_ctr_pred_model, contexts), 1))\n",
    "        current_bid_np = target_cpc*(1-is_new_index) + current_cpc*is_new_index\n",
    "\n",
    "        current_bid_np -= 10000*np.greater(total_cost, opt.budget_limit*np.ones(opt.total_ads))\n",
    "        ecpm = pCTR * current_bid_np\n",
    "        #ranking\n",
    "        max_index = np.argmax(ecpm, axis=0)\n",
    "\n",
    "        ##########################click feedback and cost reduction##########################  \n",
    "        total_impressions[max_index] += 1\n",
    "        #get the feedback\n",
    "        real_ctr = np.matmul(coefficient_ctr_model[max_index], contexts)\n",
    "        is_clicked = np.random.binomial(1, max(0, min(real_ctr, 1)))\n",
    "        if is_clicked == 1:#if clicked\n",
    "            total_clicks[max_index] += 1\n",
    "            total_cost[max_index] += target_cpc[max_index]\n",
    "\n",
    "        ##########################update pCTR prediction model##########################\n",
    "        if is_new_index[max_index]>0:#only update ctr model for new ads since we assume mature ad prediction is accurate\n",
    "            coefficient_ctr_pred_model[max_index] += opt.l_rate*(is_clicked-pCTR[max_index])*contexts#SGD for linear models\n",
    "\n",
    "        ##########################update lambda or PID##########################\n",
    "        #using PID system to update current bid\n",
    "        if i%opt.PID_cycle==(opt.PID_cycle-1):\n",
    "            current_ecpc = total_cost/(total_clicks+0.01)#compute ecpc, +0.01 to avoid 0/0=NaN\n",
    "            current_cpc = PID_sys(current_cpc, target_cpc, current_ecpc)\n",
    "        current_cpc = np.less(total_clicks, opt.target_click)*current_cpc + (1- np.less(total_clicks, opt.target_click))*target_cpc\n",
    "        \n",
    "    #Simulation results\n",
    "    print('================================================')\n",
    "    rev_new_avg_treat_group = np.sum(total_cost*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv)\n",
    "    rev_new_avg_control_group = np.sum(total_cost*is_new_control)/(np.sum(is_new_control)*opt.total_pv)\n",
    "    rev_new_tot_treat_group = np.sum(total_cost*is_new_treatment)\n",
    "    rev_new_tot_control_group = np.sum(total_cost*is_new_control)\n",
    "    print('Average Revenue in new of treat   group:', rev_new_avg_treat_group)\n",
    "    print('Average Revenue in new of control group:', rev_new_avg_control_group)\n",
    "    print('Total   Revenue in new of treat   group:', rev_new_tot_treat_group)\n",
    "    print('Total   Revenue in new of control group:', rev_new_tot_control_group)\n",
    "    \n",
    "    clicks_new_avg_treat_group = np.sum(total_clicks*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv)\n",
    "    clicks_new_avg_control_group = np.sum(total_clicks*is_new_control)/(np.sum(is_new_control)*opt.total_pv)\n",
    "    clicks_new_tot_treat_group = np.sum(total_clicks*is_new_treatment)\n",
    "    clicks_new_tot_control_group = np.sum(total_clicks*is_new_control)\n",
    "    print('Average clicks in new of treat   group:', clicks_new_avg_treat_group)\n",
    "    print('Average clicks in new of control group:', clicks_new_avg_control_group)\n",
    "    print('Total   clicks in new of treat   group:', clicks_new_tot_treat_group)\n",
    "    print('Total   clicks in new of control group:', clicks_new_tot_control_group)\n",
    "    \n",
    "    coldstart_rate_treat_group = (total_clicks*is_new_treatment>=opt.target_click).sum()/is_new_treatment.sum()\n",
    "    coldstart_rate_control_group = (total_clicks*is_new_control>=opt.target_click).sum()/is_new_control.sum()\n",
    "    print('Cold Start Success Rate of treat   group:', coldstart_rate_treat_group)\n",
    "    print('Cold Start Success Rate of control group:', coldstart_rate_control_group)\n",
    "    print('================================================')\n",
    "    return (rev_new_avg_treat_group, rev_new_avg_control_group,\n",
    "            rev_new_tot_treat_group, rev_new_tot_control_group,\n",
    "            clicks_new_avg_treat_group, clicks_new_avg_control_group,\n",
    "            clicks_new_tot_treat_group, clicks_new_tot_control_group,\n",
    "            coldstart_rate_treat_group, coldstart_rate_control_group)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.4 user-side exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_exp(opt, is_new_index, target_bid_price, coefficient_ctr_model):\n",
    "    print('User-side experiment')\n",
    "    print('# of users in treatment condition', int(opt.total_pv*opt.pv_treatment_ratio))\n",
    "    print('# of users in control condition', int(opt.total_pv*opt.pv_treatment_ratio))\n",
    "    #initialization\n",
    "    target_cpc = np.copy(target_bid_price)\n",
    "    scaled_PID_cycle_length = int(opt.PID_cycle/opt.pv_treatment_ratio)\n",
    "\n",
    "    total_clicks = np.zeros(opt.total_ads)#accumulated number of clicks\n",
    "    total_clicks_t = np.zeros(opt.total_ads)#accumulated number of clicks in treatment UVs\n",
    "    total_clicks_c = np.zeros(opt.total_ads)#accumulated number of clicks in control UVs\n",
    "    total_impressions = np.zeros(opt.total_ads)#accumulated number of impressions\n",
    "    total_impressions_t = np.zeros(opt.total_ads)#accumulated number of impressions in treatment UVs\n",
    "    total_impressions_c = np.zeros(opt.total_ads)#accumulated number of impressions in control UVs\n",
    "    total_cost = np.zeros(opt.total_ads)#accumulated cost of ads\n",
    "    total_cost_t = np.zeros(opt.total_ads)\n",
    "    total_cost_c = np.zeros(opt.total_ads)\n",
    "    coefficient_ctr_pred_model = np.zeros([opt.total_ads, dim_context]) \n",
    "\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==0:\n",
    "            coefficient_ctr_pred_model[i] = coefficient_ctr_model[i]\n",
    "        else:\n",
    "            coefficient_ctr_pred_model[i, -1] = 0.5*coefficient_ctr_model[i, -1]+0.5*0.067\n",
    "    new_ad_index = np.where(is_new_index==1)[0]\n",
    "\n",
    "    current_cpc_control = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc_control[i] = target_cpc[i] + opt.reward/opt.target_click\n",
    "\n",
    "    current_cpc_treat = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc_treat[i] = target_cpc[i] + (opt.reward+opt.delta)/opt.target_click\n",
    "\n",
    "\n",
    "    #simulation starts\n",
    "    for i in tqdm(range(opt.total_pv), desc=\"user side experiment\"):\n",
    "        ##########################generate context##########################\n",
    "        contexts = context_generate()\n",
    "        uv_condition = np.random.choice(['t', 'c', 'ow'], 1, \n",
    "                                        p=[opt.pv_treatment_ratio, opt.pv_treatment_ratio, 1-2*opt.pv_treatment_ratio])[0]\n",
    "\n",
    "        ##########################calculate eCPM and Auction##########################\n",
    "        #calculate pCTR and eCPM = pCTR*bid\n",
    "        current_bid_np = np.zeros(opt.total_ads)\n",
    "        pCTR = np.maximum(0, np.minimum(np.matmul(coefficient_ctr_pred_model, contexts), 1))\n",
    "        if uv_condition == 't':\n",
    "            current_bid_np = target_cpc*(1-is_new_index) + current_cpc_treat*is_new_index\n",
    "        elif uv_condition == 'c':\n",
    "            current_bid_np = target_cpc*(1-is_new_index) + current_cpc_control*is_new_index\n",
    "        else:\n",
    "            current_bid_np = target_cpc*(1-is_new_index) + current_cpc_control*is_new_index\n",
    "\n",
    "        current_bid_np -= 10000*np.greater(total_cost, opt.budget_limit*np.ones(opt.total_ads))\n",
    "        \n",
    "        if uv_condition == 't':\n",
    "            current_bid_np -= 10000*np.greater(total_cost_t, opt.pv_treatment_ratio*opt.budget_limit)*is_new_index\n",
    "        elif uv_condition == 'c':\n",
    "            current_bid_np -= 10000*np.greater(total_cost_c, opt.pv_treatment_ratio*opt.budget_limit)*is_new_index\n",
    "\n",
    "        ecpm = pCTR * current_bid_np\n",
    "        #ranking\n",
    "        max_index = np.argmax(ecpm, axis=0)\n",
    "\n",
    "        ##########################click feedback and cost reduction##########################  \n",
    "        total_impressions[max_index] += 1\n",
    "        if uv_condition == 't':\n",
    "            total_impressions_t[max_index] += 1\n",
    "        elif uv_condition == 'c':\n",
    "            total_impressions_c[max_index] += 1\n",
    "\n",
    "        #get the click feedback\n",
    "        real_ctr = np.matmul(coefficient_ctr_model[max_index], contexts)\n",
    "        is_clicked = np.random.binomial(1, max(0, min(real_ctr, 1)))\n",
    "        if is_clicked == 1:#if clicked\n",
    "            total_clicks[max_index] += 1\n",
    "            total_cost[max_index] += target_cpc[max_index]\n",
    "            if uv_condition == 't':\n",
    "                total_clicks_t[max_index] += 1\n",
    "                total_cost_t[max_index] += target_cpc[max_index]\n",
    "            elif uv_condition == 'c':\n",
    "                total_clicks_c[max_index] += 1\n",
    "                total_cost_c[max_index] += target_cpc[max_index]\n",
    "\n",
    "\n",
    "        ##########################update pCTR prediction model##########################\n",
    "        if is_new_index[max_index]>0:#only update ctr model for new ads since we assume mature ad prediction is accurate\n",
    "            coefficient_ctr_pred_model[max_index] += opt.l_rate*(is_clicked-pCTR[max_index])*contexts#SGD for linear models\n",
    "\n",
    "        ##########################update lambda or PID##########################\n",
    "        if i%scaled_PID_cycle_length==(scaled_PID_cycle_length-1):\n",
    "            current_ecpc = total_cost/(total_clicks+0.01)#compute ecpc, +0.01 to avoid 0/0=NaN\n",
    "            current_cpc_treat = PID_sys(current_cpc_treat,target_cpc,current_ecpc)\n",
    "            current_cpc_treat = current_cpc_treat*is_new_index\n",
    "\n",
    "            current_cpc_control = PID_sys(current_cpc_control,target_cpc,current_ecpc)\n",
    "            current_cpc_control = current_cpc_control*is_new_index\n",
    "\n",
    "        current_cpc_treat   = np.less(total_clicks_t, opt.pv_treatment_ratio*opt.target_click)*current_cpc_treat   + (1- np.less(total_clicks_t, opt.pv_treatment_ratio*opt.target_click))*target_cpc  \n",
    "        current_cpc_control = np.less(total_clicks_c, opt.pv_treatment_ratio*opt.target_click)*current_cpc_control + (1- np.less(total_clicks_c, opt.pv_treatment_ratio*opt.target_click))*target_cpc\n",
    "        \n",
    "    #Simulation results\n",
    "    print('================================================')\n",
    "    rev_new_avg_treat_group = np.sum(total_cost_t*is_new_index)/(np.sum(is_new_index)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    rev_new_avg_control_group = np.sum(total_cost_c*is_new_index)/(np.sum(is_new_index)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    rev_new_tot_treat_group = np.sum(total_cost_t*is_new_index)\n",
    "    rev_new_tot_control_group = np.sum(total_cost_c*is_new_index)\n",
    "    print('Average Revenue in new of treat   group:', rev_new_avg_treat_group)\n",
    "    print('Average Revenue in new of control group:', rev_new_avg_control_group)\n",
    "    print('Total   Revenue in new of treat   group:', rev_new_tot_treat_group)\n",
    "    print('Total   Revenue in new of control group:', rev_new_tot_control_group)\n",
    "    \n",
    "    clicks_new_avg_treat_group = np.sum(total_clicks_t*is_new_index)/(np.sum(is_new_index)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    clicks_new_avg_control_group = np.sum(total_clicks_c*is_new_index)/(np.sum(is_new_index)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    clicks_new_tot_treat_group = np.sum(total_clicks_t*is_new_index)\n",
    "    clicks_new_tot_control_group = np.sum(total_clicks_c*is_new_index)\n",
    "    print('Average clicks in new of treat   group:', clicks_new_avg_treat_group)\n",
    "    print('Average clicks in new of control group:', clicks_new_avg_control_group)\n",
    "    print('Total   clicks in new of treat   group:', clicks_new_tot_treat_group)\n",
    "    print('Total   clicks in new of control group:', clicks_new_tot_control_group)\n",
    "    \n",
    "    coldstart_rate_treat_group = (total_clicks_t*is_new_index>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_index.sum()\n",
    "    coldstart_rate_control_group = (total_clicks_c*is_new_index>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_index.sum()\n",
    "    print('Cold Start Success Rate of treat   group:', coldstart_rate_treat_group)\n",
    "    print('Cold Start Success Rate of control group:', coldstart_rate_control_group)\n",
    "    print('================================================')\n",
    "    return (rev_new_avg_treat_group, rev_new_avg_control_group,\n",
    "            rev_new_tot_treat_group, rev_new_tot_control_group,\n",
    "            clicks_new_avg_treat_group, clicks_new_avg_control_group,\n",
    "            clicks_new_tot_treat_group, clicks_new_tot_control_group,\n",
    "            coldstart_rate_treat_group, coldstart_rate_control_group)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 1.5 two-side exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def twoside_exp(opt, is_new_index, target_bid_price, coefficient_ctr_model):\n",
    "    print('Two side randomized experiment')\n",
    "    print('# of users in treatment condition', int(opt.total_pv*opt.pv_treatment_ratio))\n",
    "    print('# of users in control condition', int(opt.total_pv*opt.pv_treatment_ratio))\n",
    "    print('# of new ads in treatment condition', int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "    print('# of new ads in control condition', int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "\n",
    "    print('# of t_t pairs', int(opt.total_pv*opt.pv_treatment_ratio * opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "    print('# of t_c pairs', int(opt.total_pv*opt.pv_treatment_ratio * opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "    print('# of c_t pairs', int(opt.total_pv*opt.pv_treatment_ratio * opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "    print('# of c_c pairs', int(opt.total_pv*opt.pv_treatment_ratio * opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio))\n",
    "\n",
    "    #randomly assign new ads into treatment and control groups\n",
    "    is_new_treatment = np.zeros(opt.total_ads)\n",
    "    is_new_control = np.zeros(opt.total_ads)\n",
    "    new_ad_index = np.where(is_new_index==1)[0]\n",
    "    treatment_new_ad_index = np.random.choice(new_ad_index, int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio), replace=False)\n",
    "\n",
    "    control_new_ad_index = np.random.choice(np.setdiff1d(new_ad_index, treatment_new_ad_index), \n",
    "                                              int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio), replace=False)\n",
    "\n",
    "    for i in range(int(opt.total_ads*opt.new_ad_ratio*opt.ad_treatment_ratio)):\n",
    "        is_new_treatment[treatment_new_ad_index[i]] = 1\n",
    "        is_new_control[control_new_ad_index[i]] = 1\n",
    "\n",
    "    target_cpc = np.copy(target_bid_price)\n",
    "    scaled_PID_cycle_length = int(opt.PID_cycle/opt.pv_treatment_ratio)\n",
    "    mature_ad_index = np.where(is_new_index==0)[0]\n",
    "\n",
    "    total_impressions = np.zeros(opt.total_ads)#accumulated number of impressions\n",
    "    total_impressions_two_t = np.zeros(opt.total_ads)#accumulated number of impressions in treatment UVs & ads\n",
    "    total_impressions_two_c = np.zeros(opt.total_ads)#accumulated number of impressions in control UVs & ads\n",
    "    total_clicks = np.zeros(opt.total_ads)#accumulated number of clicks\n",
    "    total_clicks_two_t = np.zeros(opt.total_ads)#accumulated number of clicks in treatment UVs & ads\n",
    "    total_clicks_two_c = np.zeros(opt.total_ads)#accumulated number of clicks in control UVs & ads\n",
    "    total_cost = np.zeros(opt.total_ads)#accumulated cost of ads\n",
    "    total_cost_two_t = np.zeros(opt.total_ads)\n",
    "    total_cost_two_c = np.zeros(opt.total_ads)\n",
    "\n",
    "    coefficient_ctr_pred_model = np.zeros([opt.total_ads, dim_context]) \n",
    "    #assume for mature ads, we know exactly the true CTR model\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==0:\n",
    "            coefficient_ctr_pred_model[i] = coefficient_ctr_model[i]\n",
    "        else:\n",
    "            coefficient_ctr_pred_model[i, -1] = 0.5*coefficient_ctr_model[i, -1]+0.5*0.067\n",
    "\n",
    "    current_cpc_control = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc_control[i] = target_cpc[i] + opt.reward/opt.target_click\n",
    "\n",
    "    current_cpc_treat = np.zeros(opt.total_ads)\n",
    "    for i in range(opt.total_ads):\n",
    "        if is_new_index[i]==1:\n",
    "            current_cpc_treat[i] = target_cpc[i] + (opt.reward+opt.delta)/opt.target_click\n",
    "\n",
    "    #simulation starts\n",
    "    for i in tqdm(range(opt.total_pv),desc=\"two side experiment\"):\n",
    "        ##########################generate context##########################\n",
    "        contexts = context_generate()\n",
    "        uv_condition = np.random.choice(['t', 'c', 'ow'], 1, \n",
    "                                    p=[opt.pv_treatment_ratio, opt.pv_treatment_ratio, 1-2*opt.pv_treatment_ratio])[0]\n",
    "\n",
    "        ##########################calculate eCPM and Auction##########################\n",
    "        #calculate pCTR and eCPM = pCTR*bid\n",
    "        current_bid_np = np.copy(target_cpc)\n",
    "        pCTR = np.maximum(0, np.minimum(np.matmul(coefficient_ctr_pred_model, contexts), 1))\n",
    "        if uv_condition == 't':\n",
    "            current_bid_np = current_bid_np*(1-is_new_index) + current_cpc_treat*is_new_treatment + current_cpc_control*(is_new_index - is_new_treatment)\n",
    "        elif uv_condition == 'c':\n",
    "            current_bid_np = current_bid_np*(1-is_new_index) + current_cpc_control*is_new_treatment + current_cpc_control*(is_new_index - is_new_treatment)\n",
    "        else:\n",
    "            current_bid_np = current_bid_np*(1-is_new_index) + current_cpc_control*is_new_treatment + current_cpc_control*(is_new_index - is_new_treatment)\n",
    "\n",
    "        current_bid_np -= 10000*np.greater(total_cost, opt.budget_limit*np.ones(opt.total_ads))\n",
    "        \n",
    "        if uv_condition == 't':\n",
    "            current_bid_np -= 10000*np.greater(total_cost_two_t, opt.pv_treatment_ratio*opt.budget_limit)\n",
    "        elif uv_condition == 'c':\n",
    "            current_bid_np -= 10000*np.greater(total_cost_two_c, opt.pv_treatment_ratio*opt.budget_limit)\n",
    "        \n",
    "        ecpm = pCTR * current_bid_np\n",
    "        max_index = 0\n",
    "        max_index = np.argmax(ecpm, axis=0)\n",
    "\n",
    "        ##########################click feedback and cost reduction##########################  \n",
    "        total_impressions[max_index] += 1\n",
    "        if uv_condition == 't':\n",
    "            total_impressions_two_t[max_index] += 1\n",
    "        elif uv_condition == 'c':\n",
    "            total_impressions_two_c[max_index] += 1\n",
    "\n",
    "        #get the feedback\n",
    "        real_ctr = np.matmul(coefficient_ctr_model[max_index], contexts)\n",
    "        is_clicked = np.random.binomial(1, max(0, min(real_ctr, 1)))\n",
    "        if is_clicked == 1:#if clicked\n",
    "            total_clicks[max_index] += 1\n",
    "            total_cost[max_index] += target_cpc[max_index]\n",
    "            if uv_condition == 't':\n",
    "                total_clicks_two_t[max_index] += 1\n",
    "                total_cost_two_t[max_index] += target_cpc[max_index]\n",
    "            elif uv_condition == 'c':\n",
    "                total_clicks_two_c[max_index] += 1\n",
    "                total_cost_two_c[max_index] += target_cpc[max_index]\n",
    "\n",
    "\n",
    "        ##########################update pCTR prediction model##########################\n",
    "        if is_new_index[max_index]>0:#only update ctr model for new ads since we assume mature ad prediction is accurate\n",
    "            coefficient_ctr_pred_model[max_index] += opt.l_rate*(is_clicked-pCTR[max_index])*contexts#SGD for linear models\n",
    "\n",
    "        ##########################update lambda or PID##########################\n",
    "        if i%scaled_PID_cycle_length==(scaled_PID_cycle_length-1):\n",
    "            current_ecpc = total_cost/(total_clicks+0.01)#compute ecpc, +0.01 to avoid 0/0=NaN\n",
    "\n",
    "            current_cpc_treat = PID_sys(current_cpc_treat,target_cpc,current_ecpc)\n",
    "            current_cpc_treat = current_cpc_treat*is_new_index\n",
    "\n",
    "            current_cpc_control = PID_sys(current_cpc_control,target_cpc,current_ecpc)\n",
    "            current_cpc_control = current_cpc_control*is_new_index\n",
    "\n",
    "        current_cpc_treat   = np.less(total_clicks_two_t, opt.pv_treatment_ratio*opt.target_click)*current_cpc_treat   + (1- np.less(total_clicks_two_t, opt.pv_treatment_ratio*opt.target_click))*target_cpc  \n",
    "        current_cpc_control = np.less(total_clicks_two_c, opt.pv_treatment_ratio*opt.target_click)*current_cpc_control + (1- np.less(total_clicks_two_c, opt.pv_treatment_ratio*opt.target_click))*target_cpc\n",
    "\n",
    "    #Simulation results\n",
    "    print('================================================')\n",
    "    rev_new_avg_t_t = np.sum(total_cost_two_t*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    rev_new_avg_c_c = np.sum(total_cost_two_c*is_new_control)/(np.sum(is_new_control)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    rev_new_avg_t_c = np.sum(total_cost_two_t*is_new_control)/(np.sum(is_new_control)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    rev_new_avg_c_t = np.sum(total_cost_two_c*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    \n",
    "    rev_new_tot_t_t = np.sum(total_cost_two_t*is_new_treatment)\n",
    "    rev_new_tot_c_c = np.sum(total_cost_two_c*is_new_control)\n",
    "    rev_new_tot_t_c = np.sum(total_cost_two_t*is_new_control)\n",
    "    rev_new_tot_c_t = np.sum(total_cost_two_c*is_new_treatment)\n",
    "    print('Average Revenue in new of t_t group:', rev_new_avg_t_t)\n",
    "    print('Average Revenue in new of c_c group:', rev_new_avg_c_c)\n",
    "    print('Average Revenue in new of t_c group:', rev_new_avg_t_c)\n",
    "    print('Average Revenue in new of c_t group:', rev_new_avg_c_t)\n",
    "    print('Total   Revenue in new of t_t group:', rev_new_tot_t_t)\n",
    "    print('Total   Revenue in new of c_c group:', rev_new_tot_c_c)\n",
    "    print('Total   Revenue in new of t_c group:', rev_new_tot_t_c)\n",
    "    print('Total   Revenue in new of c_t group:', rev_new_tot_c_t)\n",
    "    \n",
    "    clicks_new_avg_t_t = np.sum(total_clicks_two_t*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    clicks_new_avg_c_c = np.sum(total_clicks_two_c*is_new_control)/(np.sum(is_new_control)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    clicks_new_avg_t_c = np.sum(total_clicks_two_t*is_new_control)/(np.sum(is_new_control)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "    clicks_new_avg_c_t = np.sum(total_clicks_two_c*is_new_treatment)/(np.sum(is_new_treatment)*opt.total_pv*opt.pv_treatment_ratio)\n",
    "\n",
    "    clicks_new_tot_t_t = np.sum(total_clicks_two_t*is_new_treatment)\n",
    "    clicks_new_tot_c_c = np.sum(total_clicks_two_c*is_new_control)\n",
    "    clicks_new_tot_t_c = np.sum(total_clicks_two_t*is_new_control)\n",
    "    clicks_new_tot_c_t = np.sum(total_clicks_two_c*is_new_treatment)\n",
    "    print('Average clicks in new of t_t group:', clicks_new_avg_t_t)\n",
    "    print('Average clicks in new of c_c group:', clicks_new_avg_c_c)\n",
    "    print('Average clicks in new of t_c group:', clicks_new_avg_t_c)\n",
    "    print('Average clicks in new of c_t group:', clicks_new_avg_c_t)\n",
    "    print('Total   clicks in new of t_t group:', clicks_new_tot_t_t)\n",
    "    print('Total   clicks in new of c_c group:', clicks_new_tot_c_c)\n",
    "    print('Total   clicks in new of t_c group:', clicks_new_tot_t_c)\n",
    "    print('Total   clicks in new of c_t group:', clicks_new_tot_c_t)\n",
    "    \n",
    "    coldstart_rate_t_t = (total_clicks_two_t*is_new_treatment>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_treatment.sum()\n",
    "    coldstart_rate_c_c = (total_clicks_two_c*is_new_control>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_control.sum()\n",
    "    coldstart_rate_t_c = (total_clicks_two_t*is_new_control>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_control.sum()\n",
    "    coldstart_rate_c_t = (total_clicks_two_c*is_new_treatment>=opt.pv_treatment_ratio*opt.target_click).sum()/is_new_treatment.sum()\n",
    "    print('Cold Start Success Rate of t_t group:', coldstart_rate_t_t)\n",
    "    print('Cold Start Success Rate of c_c group:', coldstart_rate_c_c)\n",
    "    print('Cold Start Success Rate of t_c group:', coldstart_rate_t_c)\n",
    "    print('Cold Start Success Rate of c_t group:', coldstart_rate_c_t)\n",
    "    print('================================================')\n",
    "    return (rev_new_avg_t_t, rev_new_avg_c_c, rev_new_avg_t_c, rev_new_avg_c_t, \n",
    "            rev_new_tot_t_t, rev_new_tot_c_c, rev_new_tot_t_c, rev_new_tot_c_t, \n",
    "            clicks_new_avg_t_t, clicks_new_avg_c_c, clicks_new_avg_t_c, clicks_new_avg_c_t,\n",
    "            clicks_new_tot_t_t, clicks_new_tot_c_c, clicks_new_tot_t_c, clicks_new_tot_c_t,\n",
    "            coldstart_rate_t_t, coldstart_rate_t_t, coldstart_rate_c_c, coldstart_rate_t_c, coldstart_rate_c_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. simulation loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['--seed'], dest='seed', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='random seed', metavar=None)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "#Default cold start parameters - the cold start value = reward*I(clicks>=target_click)\n",
    "parser.add_argument('--total_ads', type=int, default=1000, help='total number of ads for simulation')\n",
    "parser.add_argument('--new_ad_ratio', type=float, default=0.3, help='ratio of new ads')\n",
    "parser.add_argument('--total_pv', type=int, default=1000000, help='loaded total page views in one cold start horizon for simulation')\n",
    "parser.add_argument('--reward', type=float, default=2000000, help='reward of cold start')\n",
    "parser.add_argument('--delta',  type=float, default=600000, help='delta reward of treated group')\n",
    "parser.add_argument('--target_click', type=int, default=1000, help='target click')\n",
    "parser.add_argument('--alpha', type=float, default=1e-3, help='target click/total_pv')\n",
    "#If one wants to change total_ads, it is better to proportionally change total_pv to balance the two-side market\n",
    "\n",
    "#Others parameters\n",
    "parser.add_argument('--budget_limit', type=int, default=5e5, help='set an uniform budget limit on ads')\n",
    "parser.add_argument('--PID_cycle', type=int, default=100000, help='after every # pvs, update PID')\n",
    "\n",
    "#Parameters about experiments - sampling rate\n",
    "parser.add_argument('--ad_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "parser.add_argument('--pv_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "\n",
    "#Parameters about model - learning rate\n",
    "parser.add_argument('--l_rate', type=float, default=0.004, help='learning rage')\n",
    "\n",
    "#Parameters about random - random seed\n",
    "parser.add_argument('--seed', type=int, default = 1, help='random seed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'total_ads': 1000, 'new_ad_ratio': 0.3, 'total_pv': 1000000, 'reward': 2000000, 'delta': 600000, 'target_click': 1000, 'alpha': 0.001, 'budget_limit': 500000.0, 'PID_cycle': 100000, 'ad_treatment_ratio': 0.4, 'pv_treatment_ratio': 0.4, 'l_rate': 0.004, 'seed': 1}\n"
     ]
    }
   ],
   "source": [
    "opt = parser.parse_args()\n",
    "np.save('my_config_seed{}.npy'.format(opt.seed), opt)\n",
    "print(opt.__dict__)\n",
    "np.random.seed(opt.seed)\n",
    "is_new_index, target_bid_price, coefficient_ctr_model = generate_ad(n = opt.total_ads, new_raito = opt.new_ad_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29b5630cad4d496d8570281dc014bdd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Simulation progress:   0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "090ad2e4e94544b6aa464ee731252829",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.0196159929420868\n",
      "Total   Revenue in new: 5884797.88262604\n",
      "Average clicks in new: 0.00017581666666666667\n",
      "Total   clicks in new: 52745.0\n",
      "Cold Start Success Rate: 0.15666666666666668\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7c06855deb74ae683518f11f4044f11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.019890403700454855\n",
      "Total   Revenue in new: 5967121.110136457\n",
      "Average clicks in new: 0.00019193\n",
      "Total   clicks in new: 57579.0\n",
      "Cold Start Success Rate: 0.17\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d580449cd354d488fca2e6485dbb141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.02281341516822458\n",
      "Average Revenue in new of control group: 0.01387995525973929\n",
      "Total   Revenue in new of treat   group: 2737609.82018695\n",
      "Total   Revenue in new of control group: 1665594.6311687147\n",
      "Average clicks in new of treat   group: 0.00029698333333333333\n",
      "Average clicks in new of control group: 0.000100325\n",
      "Total   clicks in new of treat   group: 35638.0\n",
      "Total   clicks in new of control group: 12039.0\n",
      "Cold Start Success Rate of treat   group: 0.24166666666666667\n",
      "Cold Start Success Rate of control group: 0.08333333333333333\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "512df5d4846b44719861a5615b59e157",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03496665520269376\n",
      "Average Revenue in new of control group: 0.03505989906609438\n",
      "Total   Revenue in new of treat   group: 4195998.624323251\n",
      "Total   Revenue in new of control group: 4207187.8879313255\n",
      "Average clicks in new of treat   group: 0.00035998333333333335\n",
      "Average clicks in new of control group: 0.000358525\n",
      "Total   clicks in new of treat   group: 43198.0\n",
      "Total   clicks in new of control group: 43023.0\n",
      "Cold Start Success Rate of treat   group: 0.34\n",
      "Cold Start Success Rate of control group: 0.33666666666666667\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccedb340e8a6436384c3b2f2b59d5801",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.04643326939819097\n",
      "Average Revenue in new of c_c group: 0.020115440641335074\n",
      "Average Revenue in new of t_c group: 0.01742676047805749\n",
      "Average Revenue in new of c_t group: 0.04353004834838293\n",
      "Total   Revenue in new of t_t group: 2228796.9311131667\n",
      "Total   Revenue in new of c_c group: 965541.1507840835\n",
      "Total   Revenue in new of t_c group: 836484.5029467595\n",
      "Total   Revenue in new of c_t group: 2089442.3207223804\n",
      "Average clicks in new of t_t group: 0.00046335416666666666\n",
      "Average clicks in new of c_c group: 0.00030604166666666665\n",
      "Average clicks in new of t_c group: 0.00026235416666666665\n",
      "Average clicks in new of c_t group: 0.0004420625\n",
      "Total   clicks in new of t_t group: 22241.0\n",
      "Total   clicks in new of c_c group: 14690.0\n",
      "Total   clicks in new of t_c group: 12593.0\n",
      "Total   clicks in new of c_t group: 21219.0\n",
      "Cold Start Success Rate of t_t group: 0.425\n",
      "Cold Start Success Rate of c_c group: 0.26666666666666666\n",
      "Cold Start Success Rate of t_c group: 0.041666666666666664\n",
      "Cold Start Success Rate of c_t group: 0.4\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54d8cc17b84e4e47b247a64e7d62f58f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.011525985669496896\n",
      "Total   Revenue in new: 3457795.700849069\n",
      "Average clicks in new: 0.0001745\n",
      "Total   clicks in new: 52350.0\n",
      "Cold Start Success Rate: 0.16666666666666666\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3875ae4e9b542c7b82477c051039c3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.013715212768163666\n",
      "Total   Revenue in new: 4114563.8304490997\n",
      "Average clicks in new: 0.00020002\n",
      "Total   clicks in new: 60006.0\n",
      "Cold Start Success Rate: 0.18666666666666668\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13130acd5e254bcdb6d2eaa7b575343d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.02359818984167132\n",
      "Average Revenue in new of control group: 0.00788508688710845\n",
      "Total   Revenue in new of treat   group: 2831782.7810005588\n",
      "Total   Revenue in new of control group: 946210.426453014\n",
      "Average clicks in new of treat   group: 0.00028359166666666667\n",
      "Average clicks in new of control group: 0.00011520833333333333\n",
      "Total   clicks in new of treat   group: 34031.0\n",
      "Total   clicks in new of control group: 13825.0\n",
      "Cold Start Success Rate of treat   group: 0.25\n",
      "Cold Start Success Rate of control group: 0.1\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3978e8be1884db1922f0f9b85e9050c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03399736916560962\n",
      "Average Revenue in new of control group: 0.0337471038143853\n",
      "Total   Revenue in new of treat   group: 4079684.2998731546\n",
      "Total   Revenue in new of control group: 4049652.457726236\n",
      "Average clicks in new of treat   group: 0.00040894166666666665\n",
      "Average clicks in new of control group: 0.00040428333333333334\n",
      "Total   clicks in new of treat   group: 49073.0\n",
      "Total   clicks in new of control group: 48514.0\n",
      "Cold Start Success Rate of treat   group: 0.38666666666666666\n",
      "Cold Start Success Rate of control group: 0.37333333333333335\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "740c0226cdf24412b79220cb4b7b63ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.050694892840318535\n",
      "Average Revenue in new of c_c group: 0.017872832935641746\n",
      "Average Revenue in new of t_c group: 0.014686222076051577\n",
      "Average Revenue in new of c_t group: 0.04899092159794487\n",
      "Total   Revenue in new of t_t group: 2433354.85633529\n",
      "Total   Revenue in new of c_c group: 857895.9809108039\n",
      "Total   Revenue in new of t_c group: 704938.6596504757\n",
      "Total   Revenue in new of c_t group: 2351564.2367013535\n",
      "Average clicks in new of t_t group: 0.0005675208333333333\n",
      "Average clicks in new of c_c group: 0.0002574375\n",
      "Average clicks in new of t_c group: 0.00021145833333333333\n",
      "Average clicks in new of c_t group: 0.0005422708333333333\n",
      "Total   clicks in new of t_t group: 27241.0\n",
      "Total   clicks in new of c_c group: 12357.0\n",
      "Total   clicks in new of t_c group: 10150.0\n",
      "Total   clicks in new of c_t group: 26029.0\n",
      "Cold Start Success Rate of t_t group: 0.525\n",
      "Cold Start Success Rate of c_c group: 0.23333333333333334\n",
      "Cold Start Success Rate of t_c group: 0.025\n",
      "Cold Start Success Rate of c_t group: 0.5083333333333333\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de0c001bbf5f4a68843a04d6302c8433",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.01516967376280423\n",
      "Total   Revenue in new: 4550902.128841269\n",
      "Average clicks in new: 0.00017119\n",
      "Total   clicks in new: 51357.0\n",
      "Cold Start Success Rate: 0.15333333333333332\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3984afbcbc2a42938efcded7724c16d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.02102485775224741\n",
      "Total   Revenue in new: 6307457.325674224\n",
      "Average clicks in new: 0.00019800666666666667\n",
      "Total   clicks in new: 59402.0\n",
      "Cold Start Success Rate: 0.18\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8399e6b4f9db405bb2b4f0d4f44a358f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.020499104232729774\n",
      "Average Revenue in new of control group: 0.012842391093626095\n",
      "Total   Revenue in new of treat   group: 2459892.507927573\n",
      "Total   Revenue in new of control group: 1541086.9312351313\n",
      "Average clicks in new of treat   group: 0.00028261666666666666\n",
      "Average clicks in new of control group: 9.010833333333333e-05\n",
      "Total   clicks in new of treat   group: 33914.0\n",
      "Total   clicks in new of control group: 10813.0\n",
      "Cold Start Success Rate of treat   group: 0.25\n",
      "Cold Start Success Rate of control group: 0.08333333333333333\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f8d1fe5c17549448709d8ddb7d1e405",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.038562337784627\n",
      "Average Revenue in new of control group: 0.03860083658726109\n",
      "Total   Revenue in new of treat   group: 4627480.534155239\n",
      "Total   Revenue in new of control group: 4632100.390471331\n",
      "Average clicks in new of treat   group: 0.000379175\n",
      "Average clicks in new of control group: 0.00037831666666666666\n",
      "Total   clicks in new of treat   group: 45501.0\n",
      "Total   clicks in new of control group: 45398.0\n",
      "Cold Start Success Rate of treat   group: 0.36\n",
      "Cold Start Success Rate of control group: 0.3566666666666667\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdd1876748c84e2d80d3f1de808b4572",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.04518538956566748\n",
      "Average Revenue in new of c_c group: 0.027708911141682828\n",
      "Average Revenue in new of t_c group: 0.02430885257278867\n",
      "Average Revenue in new of c_t group: 0.04301364900159669\n",
      "Total   Revenue in new of t_t group: 2168898.699152039\n",
      "Total   Revenue in new of c_c group: 1330027.7348007758\n",
      "Total   Revenue in new of t_c group: 1166824.923493856\n",
      "Total   Revenue in new of c_t group: 2064655.152076641\n",
      "Average clicks in new of t_t group: 0.0004721041666666667\n",
      "Average clicks in new of c_c group: 0.00030329166666666666\n",
      "Average clicks in new of t_c group: 0.00026083333333333335\n",
      "Average clicks in new of c_t group: 0.0004500833333333333\n",
      "Total   clicks in new of t_t group: 22661.0\n",
      "Total   clicks in new of c_c group: 14558.0\n",
      "Total   clicks in new of t_c group: 12520.0\n",
      "Total   clicks in new of c_t group: 21604.0\n",
      "Cold Start Success Rate of t_t group: 0.45\n",
      "Cold Start Success Rate of c_c group: 0.2833333333333333\n",
      "Cold Start Success Rate of t_c group: 0.016666666666666666\n",
      "Cold Start Success Rate of c_t group: 0.4\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92ba4612f1944655948b36d6c590be38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.012966093419913218\n",
      "Total   Revenue in new: 3889828.0259739654\n",
      "Average clicks in new: 0.00017197666666666668\n",
      "Total   clicks in new: 51593.0\n",
      "Cold Start Success Rate: 0.15666666666666668\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0afa32a422a54d2bb3974f34058dbfce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.014239615848147254\n",
      "Total   Revenue in new: 4271884.754444176\n",
      "Average clicks in new: 0.00018641\n",
      "Total   clicks in new: 55923.0\n",
      "Cold Start Success Rate: 0.17\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a92c8509896469293e55637456c704e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.029655441571515808\n",
      "Average Revenue in new of control group: 0.006753963045608116\n",
      "Total   Revenue in new of treat   group: 3558652.9885818968\n",
      "Total   Revenue in new of control group: 810475.565472974\n",
      "Average clicks in new of treat   group: 0.00027579166666666664\n",
      "Average clicks in new of control group: 0.00010398333333333333\n",
      "Total   clicks in new of treat   group: 33095.0\n",
      "Total   clicks in new of control group: 12478.0\n",
      "Cold Start Success Rate of treat   group: 0.25\n",
      "Cold Start Success Rate of control group: 0.08333333333333333\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d13d7e0cbfc4d5f8774b87c54dc2795",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03448026165784557\n",
      "Average Revenue in new of control group: 0.03404958223939161\n",
      "Total   Revenue in new of treat   group: 4137631.3989414684\n",
      "Total   Revenue in new of control group: 4085949.868726994\n",
      "Average clicks in new of treat   group: 0.000392975\n",
      "Average clicks in new of control group: 0.000388975\n",
      "Total   clicks in new of treat   group: 47157.0\n",
      "Total   clicks in new of control group: 46677.0\n",
      "Cold Start Success Rate of treat   group: 0.36\n",
      "Cold Start Success Rate of control group: 0.35\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2099fe47b19945f2b583c6928ef0b3e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.039400975950645886\n",
      "Average Revenue in new of c_c group: 0.02246580547480389\n",
      "Average Revenue in new of t_c group: 0.02012522003148589\n",
      "Average Revenue in new of c_t group: 0.0380637082684665\n",
      "Total   Revenue in new of t_t group: 1891246.8456310024\n",
      "Total   Revenue in new of c_c group: 1078358.6627905867\n",
      "Total   Revenue in new of t_c group: 966010.5615113227\n",
      "Total   Revenue in new of c_t group: 1827057.996886392\n",
      "Average clicks in new of t_t group: 0.0004693333333333333\n",
      "Average clicks in new of c_c group: 0.0002513125\n",
      "Average clicks in new of t_c group: 0.00021975\n",
      "Average clicks in new of c_t group: 0.0004554375\n",
      "Total   clicks in new of t_t group: 22528.0\n",
      "Total   clicks in new of c_c group: 12063.0\n",
      "Total   clicks in new of t_c group: 10548.0\n",
      "Total   clicks in new of c_t group: 21861.0\n",
      "Cold Start Success Rate of t_t group: 0.44166666666666665\n",
      "Cold Start Success Rate of c_c group: 0.20833333333333334\n",
      "Cold Start Success Rate of t_c group: 0.05\n",
      "Cold Start Success Rate of c_t group: 0.4\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "301ce40ea9c643979ad1e26bcad3e5ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.017222335952011093\n",
      "Total   Revenue in new: 5166700.785603328\n",
      "Average clicks in new: 0.00017008\n",
      "Total   clicks in new: 51024.0\n",
      "Cold Start Success Rate: 0.15333333333333332\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0a9c92df6b748e88ed3e2c7d15ce934",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.019542358320177672\n",
      "Total   Revenue in new: 5862707.496053302\n",
      "Average clicks in new: 0.00019371333333333333\n",
      "Total   clicks in new: 58114.0\n",
      "Cold Start Success Rate: 0.16666666666666666\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61fee60508084d4c9d2eaceca0e0cdf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.017578923987629447\n",
      "Average Revenue in new of control group: 0.014144431609735067\n",
      "Total   Revenue in new of treat   group: 2109470.8785155336\n",
      "Total   Revenue in new of control group: 1697331.793168208\n",
      "Average clicks in new of treat   group: 0.00022933333333333334\n",
      "Average clicks in new of control group: 0.00012854166666666667\n",
      "Total   clicks in new of treat   group: 27520.0\n",
      "Total   clicks in new of control group: 15425.0\n",
      "Cold Start Success Rate of treat   group: 0.20833333333333334\n",
      "Cold Start Success Rate of control group: 0.11666666666666667\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfe01973e318463083dc1224193b8873",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03933014711114507\n",
      "Average Revenue in new of control group: 0.03896814795329575\n",
      "Total   Revenue in new of treat   group: 4719617.653337408\n",
      "Total   Revenue in new of control group: 4676177.75439549\n",
      "Average clicks in new of treat   group: 0.00039063333333333336\n",
      "Average clicks in new of control group: 0.0003856833333333333\n",
      "Total   clicks in new of treat   group: 46876.0\n",
      "Total   clicks in new of control group: 46282.0\n",
      "Cold Start Success Rate of treat   group: 0.3566666666666667\n",
      "Cold Start Success Rate of control group: 0.3566666666666667\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b60610bdd742e383fa4b0db5bf224e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.04160648403932349\n",
      "Average Revenue in new of c_c group: 0.032016057558495084\n",
      "Average Revenue in new of t_c group: 0.028655893672749254\n",
      "Average Revenue in new of c_t group: 0.039316619335357884\n",
      "Total   Revenue in new of t_t group: 1997111.2338875274\n",
      "Total   Revenue in new of c_c group: 1536770.7628077639\n",
      "Total   Revenue in new of t_c group: 1375482.8962919642\n",
      "Total   Revenue in new of c_t group: 1887197.7280971785\n",
      "Average clicks in new of t_t group: 0.00041235416666666666\n",
      "Average clicks in new of c_c group: 0.00032564583333333333\n",
      "Average clicks in new of t_c group: 0.0002900416666666667\n",
      "Average clicks in new of c_t group: 0.0003839166666666667\n",
      "Total   clicks in new of t_t group: 19793.0\n",
      "Total   clicks in new of c_c group: 15631.0\n",
      "Total   clicks in new of t_c group: 13922.0\n",
      "Total   clicks in new of c_t group: 18428.0\n",
      "Cold Start Success Rate of t_t group: 0.36666666666666664\n",
      "Cold Start Success Rate of c_c group: 0.2916666666666667\n",
      "Cold Start Success Rate of t_c group: 0.06666666666666667\n",
      "Cold Start Success Rate of c_t group: 0.3416666666666667\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b6efca1aa1f4f5f9722b50d1d2551dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.01745074323021368\n",
      "Total   Revenue in new: 5235222.969064103\n",
      "Average clicks in new: 0.00017004333333333333\n",
      "Total   clicks in new: 51013.0\n",
      "Cold Start Success Rate: 0.15\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb54875092634ce296ca49a7e41556fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.019425534617172485\n",
      "Total   Revenue in new: 5827660.385151746\n",
      "Average clicks in new: 0.00019972666666666667\n",
      "Total   clicks in new: 59918.0\n",
      "Cold Start Success Rate: 0.18\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2066d65ae4ea4f2bb5898a3bc2d85c8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03316677609276932\n",
      "Average Revenue in new of control group: 0.008155597474525673\n",
      "Total   Revenue in new of treat   group: 3980013.1311323186\n",
      "Total   Revenue in new of control group: 978671.6969430806\n",
      "Average clicks in new of treat   group: 0.000288825\n",
      "Average clicks in new of control group: 8.275e-05\n",
      "Total   clicks in new of treat   group: 34659.0\n",
      "Total   clicks in new of control group: 9930.0\n",
      "Cold Start Success Rate of treat   group: 0.25\n",
      "Cold Start Success Rate of control group: 0.058333333333333334\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6bf54d096f54c0eb475f46f48d91e9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.04088406084493521\n",
      "Average Revenue in new of control group: 0.041152399868651816\n",
      "Total   Revenue in new of treat   group: 4906087.301392226\n",
      "Total   Revenue in new of control group: 4938287.984238218\n",
      "Average clicks in new of treat   group: 0.00039335833333333333\n",
      "Average clicks in new of control group: 0.00039145\n",
      "Total   clicks in new of treat   group: 47203.0\n",
      "Total   clicks in new of control group: 46974.0\n",
      "Cold Start Success Rate of treat   group: 0.36\n",
      "Cold Start Success Rate of control group: 0.35333333333333333\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d24af7923b0840609a6e047fda7e6591",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.04863814091192963\n",
      "Average Revenue in new of c_c group: 0.024867899252585052\n",
      "Average Revenue in new of t_c group: 0.02166114062886048\n",
      "Average Revenue in new of c_t group: 0.046985784848977814\n",
      "Total   Revenue in new of t_t group: 2334630.763772622\n",
      "Total   Revenue in new of c_c group: 1193659.1641240825\n",
      "Total   Revenue in new of t_c group: 1039734.750185303\n",
      "Total   Revenue in new of c_t group: 2255317.672750935\n",
      "Average clicks in new of t_t group: 0.0004948125\n",
      "Average clicks in new of c_c group: 0.0002356875\n",
      "Average clicks in new of t_c group: 0.00020164583333333333\n",
      "Average clicks in new of c_t group: 0.00047064583333333333\n",
      "Total   clicks in new of t_t group: 23751.0\n",
      "Total   clicks in new of c_c group: 11313.0\n",
      "Total   clicks in new of t_c group: 9679.0\n",
      "Total   clicks in new of c_t group: 22591.0\n",
      "Cold Start Success Rate of t_t group: 0.44166666666666665\n",
      "Cold Start Success Rate of c_c group: 0.2\n",
      "Cold Start Success Rate of t_c group: 0.03333333333333333\n",
      "Cold Start Success Rate of c_t group: 0.4166666666666667\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f576efb9809149e9b1020929d16826d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.012614576614462048\n",
      "Total   Revenue in new: 3784372.984338614\n",
      "Average clicks in new: 0.00015831666666666667\n",
      "Total   clicks in new: 47495.0\n",
      "Cold Start Success Rate: 0.14666666666666667\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50dd27489e9341e5aaffe874f1ddcc75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.015542547025071697\n",
      "Total   Revenue in new: 4662764.107521509\n",
      "Average clicks in new: 0.00018619\n",
      "Total   clicks in new: 55857.0\n",
      "Cold Start Success Rate: 0.17\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "738e42356c474e98bf8767a041556d23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.022866822816291807\n",
      "Average Revenue in new of control group: 0.01229636905282092\n",
      "Total   Revenue in new of treat   group: 2744018.737955017\n",
      "Total   Revenue in new of control group: 1475564.2863385102\n",
      "Average clicks in new of treat   group: 0.00022605833333333333\n",
      "Average clicks in new of control group: 0.00013969166666666667\n",
      "Total   clicks in new of treat   group: 27127.0\n",
      "Total   clicks in new of control group: 16763.0\n",
      "Cold Start Success Rate of treat   group: 0.2\n",
      "Cold Start Success Rate of control group: 0.125\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b84c8d59311b403bad0fb2e6b7c90280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.027356152029463416\n",
      "Average Revenue in new of control group: 0.027375109887094306\n",
      "Total   Revenue in new of treat   group: 3282738.24353561\n",
      "Total   Revenue in new of control group: 3285013.186451317\n",
      "Average clicks in new of treat   group: 0.000355525\n",
      "Average clicks in new of control group: 0.0003538833333333333\n",
      "Total   clicks in new of treat   group: 42663.0\n",
      "Total   clicks in new of control group: 42466.0\n",
      "Cold Start Success Rate of treat   group: 0.32\n",
      "Cold Start Success Rate of control group: 0.31666666666666665\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ee7197953cc44afbdbd46c36dabb7ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.03320888187236295\n",
      "Average Revenue in new of c_c group: 0.026294961766764147\n",
      "Average Revenue in new of t_c group: 0.024695835862036997\n",
      "Average Revenue in new of c_t group: 0.031478232579151486\n",
      "Total   Revenue in new of t_t group: 1594026.3298734215\n",
      "Total   Revenue in new of c_c group: 1262158.164804679\n",
      "Total   Revenue in new of t_c group: 1185400.121377776\n",
      "Total   Revenue in new of c_t group: 1510955.1637992715\n",
      "Average clicks in new of t_t group: 0.000371125\n",
      "Average clicks in new of c_c group: 0.0003276875\n",
      "Average clicks in new of t_c group: 0.00030083333333333335\n",
      "Average clicks in new of c_t group: 0.00034970833333333333\n",
      "Total   clicks in new of t_t group: 17814.0\n",
      "Total   clicks in new of c_c group: 15729.0\n",
      "Total   clicks in new of t_c group: 14440.0\n",
      "Total   clicks in new of c_t group: 16786.0\n",
      "Cold Start Success Rate of t_t group: 0.3333333333333333\n",
      "Cold Start Success Rate of c_c group: 0.30833333333333335\n",
      "Cold Start Success Rate of t_c group: 0.075\n",
      "Cold Start Success Rate of c_t group: 0.31666666666666665\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c01fe0ff93c4976925cb0ef6ef3e71b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.017144913455849245\n",
      "Total   Revenue in new: 5143474.036754773\n",
      "Average clicks in new: 0.00017911666666666667\n",
      "Total   clicks in new: 53735.0\n",
      "Cold Start Success Rate: 0.16333333333333333\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "090e3678af8d48869aac746ac94f8aaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.02018948934153134\n",
      "Total   Revenue in new: 6056846.802459402\n",
      "Average clicks in new: 0.00020024\n",
      "Total   clicks in new: 60072.0\n",
      "Cold Start Success Rate: 0.17666666666666667\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20cabbbcc5734e08a378216d2103ce15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.02351356548106572\n",
      "Average Revenue in new of control group: 0.011211092551446904\n",
      "Total   Revenue in new of treat   group: 2821627.857727886\n",
      "Total   Revenue in new of control group: 1345331.1061736285\n",
      "Average clicks in new of treat   group: 0.0002971333333333333\n",
      "Average clicks in new of control group: 9.940833333333333e-05\n",
      "Total   clicks in new of treat   group: 35656.0\n",
      "Total   clicks in new of control group: 11929.0\n",
      "Cold Start Success Rate of treat   group: 0.24166666666666667\n",
      "Cold Start Success Rate of control group: 0.08333333333333333\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37911a418eff4bea8d041649c43d3b32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.03140589155980519\n",
      "Average Revenue in new of control group: 0.03150014127221966\n",
      "Total   Revenue in new of treat   group: 3768706.9871766223\n",
      "Total   Revenue in new of control group: 3780016.952666359\n",
      "Average clicks in new of treat   group: 0.0003635166666666667\n",
      "Average clicks in new of control group: 0.0003621\n",
      "Total   clicks in new of treat   group: 43622.0\n",
      "Total   clicks in new of control group: 43452.0\n",
      "Cold Start Success Rate of treat   group: 0.3466666666666667\n",
      "Cold Start Success Rate of control group: 0.34\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "134e02632af1462ebc9bb7bd069e29dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.047251882049124685\n",
      "Average Revenue in new of c_c group: 0.03053297964424979\n",
      "Average Revenue in new of t_c group: 0.02839320801644785\n",
      "Average Revenue in new of c_t group: 0.045068740758111146\n",
      "Total   Revenue in new of t_t group: 2268090.338357985\n",
      "Total   Revenue in new of c_c group: 1465583.02292399\n",
      "Total   Revenue in new of t_c group: 1362873.9847894968\n",
      "Total   Revenue in new of c_t group: 2163299.556389335\n",
      "Average clicks in new of t_t group: 0.0004762083333333333\n",
      "Average clicks in new of c_c group: 0.00032333333333333335\n",
      "Average clicks in new of t_c group: 0.0002920625\n",
      "Average clicks in new of c_t group: 0.000452125\n",
      "Total   clicks in new of t_t group: 22858.0\n",
      "Total   clicks in new of c_c group: 15520.0\n",
      "Total   clicks in new of t_c group: 14019.0\n",
      "Total   clicks in new of c_t group: 21702.0\n",
      "Cold Start Success Rate of t_t group: 0.44166666666666665\n",
      "Cold Start Success Rate of c_c group: 0.3\n",
      "Cold Start Success Rate of t_c group: 0.06666666666666667\n",
      "Cold Start Success Rate of c_t group: 0.4\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64d7595de35940eb868bcfe17417fb18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.012014811496405937\n",
      "Total   Revenue in new: 3604443.448921781\n",
      "Average clicks in new: 0.00017166\n",
      "Total   clicks in new: 51498.0\n",
      "Cold Start Success Rate: 0.14333333333333334\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c3b0392cdf94a10b9966061946bee9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.013849444723638306\n",
      "Total   Revenue in new: 4154833.417091492\n",
      "Average clicks in new: 0.00019931666666666667\n",
      "Total   clicks in new: 59795.0\n",
      "Cold Start Success Rate: 0.18333333333333332\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2702651c69384849a603b649ce80c6e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.017109307409349853\n",
      "Average Revenue in new of control group: 0.004663648409710343\n",
      "Total   Revenue in new of treat   group: 2053116.8891219825\n",
      "Total   Revenue in new of control group: 559637.8091652412\n",
      "Average clicks in new of treat   group: 0.00021303333333333333\n",
      "Average clicks in new of control group: 0.000126725\n",
      "Total   clicks in new of treat   group: 25564.0\n",
      "Total   clicks in new of control group: 15207.0\n",
      "Cold Start Success Rate of treat   group: 0.19166666666666668\n",
      "Cold Start Success Rate of control group: 0.10833333333333334\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d1d7293b41d49ee9c0da7ddae0f330c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.029557566241015923\n",
      "Average Revenue in new of control group: 0.02927783819315993\n",
      "Total   Revenue in new of treat   group: 3546907.948921911\n",
      "Total   Revenue in new of control group: 3513340.5831791917\n",
      "Average clicks in new of treat   group: 0.00038844166666666664\n",
      "Average clicks in new of control group: 0.000386\n",
      "Total   clicks in new of treat   group: 46613.0\n",
      "Total   clicks in new of control group: 46320.0\n",
      "Cold Start Success Rate of treat   group: 0.36333333333333334\n",
      "Cold Start Success Rate of control group: 0.36333333333333334\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e85e503ecb84338a1710ebf7f9e524e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.03234547579332716\n",
      "Average Revenue in new of c_c group: 0.02276368019311719\n",
      "Average Revenue in new of t_c group: 0.020328911985808606\n",
      "Average Revenue in new of c_t group: 0.030624266950484143\n",
      "Total   Revenue in new of t_t group: 1552582.8380797035\n",
      "Total   Revenue in new of c_c group: 1092656.649269625\n",
      "Total   Revenue in new of t_c group: 975787.775318813\n",
      "Total   Revenue in new of c_t group: 1469964.8136232388\n",
      "Average clicks in new of t_t group: 0.00042745833333333335\n",
      "Average clicks in new of c_c group: 0.00032908333333333335\n",
      "Average clicks in new of t_c group: 0.0002837916666666667\n",
      "Average clicks in new of c_t group: 0.00040410416666666665\n",
      "Total   clicks in new of t_t group: 20518.0\n",
      "Total   clicks in new of c_c group: 15796.0\n",
      "Total   clicks in new of t_c group: 13622.0\n",
      "Total   clicks in new of c_t group: 19397.0\n",
      "Cold Start Success Rate of t_t group: 0.39166666666666666\n",
      "Cold Start Success Rate of c_c group: 0.2916666666666667\n",
      "Cold Start Success Rate of t_c group: 0.041666666666666664\n",
      "Cold Start Success Rate of c_t group: 0.36666666666666664\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94345e99af444f888c52c57867739ae0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.015110250323939199\n",
      "Total   Revenue in new: 4533075.09718176\n",
      "Average clicks in new: 0.00016573\n",
      "Total   clicks in new: 49719.0\n",
      "Cold Start Success Rate: 0.14333333333333334\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d723baf437e242ccb64e0e48f259e6dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.017954630462009492\n",
      "Total   Revenue in new: 5386389.138602848\n",
      "Average clicks in new: 0.00019178\n",
      "Total   clicks in new: 57534.0\n",
      "Cold Start Success Rate: 0.16333333333333333\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1096db05a0da48c48f2de3771508fb4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.02170903150173386\n",
      "Average Revenue in new of control group: 0.009244291423588137\n",
      "Total   Revenue in new of treat   group: 2605083.7802080633\n",
      "Total   Revenue in new of control group: 1109314.9708305765\n",
      "Average clicks in new of treat   group: 0.00025869166666666666\n",
      "Average clicks in new of control group: 9.2225e-05\n",
      "Total   clicks in new of treat   group: 31043.0\n",
      "Total   clicks in new of control group: 11067.0\n",
      "Cold Start Success Rate of treat   group: 0.23333333333333334\n",
      "Cold Start Success Rate of control group: 0.075\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "736474ba1f3247b0b98872a6110f1ba7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.032459244853349015\n",
      "Average Revenue in new of control group: 0.032193502716273234\n",
      "Total   Revenue in new of treat   group: 3895109.3824018817\n",
      "Total   Revenue in new of control group: 3863220.3259527883\n",
      "Average clicks in new of treat   group: 0.00036599166666666667\n",
      "Average clicks in new of control group: 0.00036149166666666667\n",
      "Total   clicks in new of treat   group: 43919.0\n",
      "Total   clicks in new of control group: 43379.0\n",
      "Cold Start Success Rate of treat   group: 0.3333333333333333\n",
      "Cold Start Success Rate of control group: 0.32666666666666666\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 120\n",
      "# of new ads in control condition 120\n",
      "# of t_t pairs 48000000\n",
      "# of t_c pairs 48000000\n",
      "# of c_t pairs 48000000\n",
      "# of c_c pairs 48000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b210b2cba0f74d04ac21bb93e0849bb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.055758200911067785\n",
      "Average Revenue in new of c_c group: 0.01349859388423075\n",
      "Average Revenue in new of t_c group: 0.011729835731146943\n",
      "Average Revenue in new of c_t group: 0.053961179786373374\n",
      "Total   Revenue in new of t_t group: 2676393.6437312537\n",
      "Total   Revenue in new of c_c group: 647932.5064430761\n",
      "Total   Revenue in new of t_c group: 563032.1150950532\n",
      "Total   Revenue in new of c_t group: 2590136.629745922\n",
      "Average clicks in new of t_t group: 0.0005026041666666667\n",
      "Average clicks in new of c_c group: 0.000240625\n",
      "Average clicks in new of t_c group: 0.00020045833333333333\n",
      "Average clicks in new of c_t group: 0.0004824375\n",
      "Total   clicks in new of t_t group: 24125.0\n",
      "Total   clicks in new of c_c group: 11550.0\n",
      "Total   clicks in new of t_c group: 9622.0\n",
      "Total   clicks in new of c_t group: 23157.0\n",
      "Cold Start Success Rate of t_t group: 0.4666666666666667\n",
      "Cold Start Success Rate of c_c group: 0.225\n",
      "Cold Start Success Rate of t_c group: 0.008333333333333333\n",
      "Cold Start Success Rate of c_t group: 0.43333333333333335\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\276870088.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "# Initialize an empty DataFrame to store the results\n",
    "metrecs = [\"rev_new_avg\",\"rev_new_tot\",\"clicks_new_avg\",'clicks_new_tot','coldstart_rate']\n",
    "exp = ['_all_treat', '_all_control', \n",
    "       '_ad_treat', '_ad_control', \n",
    "       '_user_treat', '_user_control', \n",
    "       '_t_t', '_c_c', '_t_c', '_c_t']\n",
    "columns = [i+j for i in metrecs for j in exp ]\n",
    "results_df = pd.DataFrame(columns = columns)\n",
    "\n",
    "# Run the simulation loop 10 times\n",
    "for i in tqdm(range(10), desc=\"Simulation progress\"):\n",
    "    opt = parser.parse_args()\n",
    "    is_new_index, target_bid_price, coefficient_ctr_model \\\n",
    "    = generate_ad(n = opt.total_ads, new_raito = opt.new_ad_ratio)\n",
    "\n",
    "    rev_new_avg_all_control, rev_new_tot_all_control, \\\n",
    "    clicks_new_avg_all_control, clicks_new_tot_all_control, \\\n",
    "    coldstart_rate_all_control = all_control(opt = opt, is_new_index = is_new_index , \n",
    "                                target_bid_price = target_bid_price, \n",
    "                                coefficient_ctr_model = coefficient_ctr_model)\n",
    "    \n",
    "    rev_new_avg_all_treat,   rev_new_tot_all_treat,   \\\n",
    "    clicks_new_avg_all_treat,   clicks_new_tot_all_treat,   \\\n",
    "    coldstart_rate_all_treat  = all_treat(opt = opt, is_new_index = is_new_index , \n",
    "                                target_bid_price = target_bid_price, \n",
    "                                coefficient_ctr_model = coefficient_ctr_model)\n",
    "\n",
    "    rev_new_avg_ad_treat, rev_new_avg_ad_control, \\\n",
    "    rev_new_tot_ad_treat, rev_new_tot_ad_control, \\\n",
    "    clicks_new_avg_ad_treat, clicks_new_avg_ad_control, \\\n",
    "    clicks_new_tot_ad_treat, clicks_new_tot_ad_control, \\\n",
    "    coldstart_rate_ad_treat, coldstart_rate_ad_control \\\n",
    "    = ad_exp(opt = opt, is_new_index = is_new_index , \n",
    "             target_bid_price = target_bid_price, \n",
    "             coefficient_ctr_model = coefficient_ctr_model) \n",
    "\n",
    "    rev_new_avg_user_treat, rev_new_avg_user_control, \\\n",
    "    rev_new_tot_user_treat, rev_new_tot_user_control, \\\n",
    "    clicks_new_avg_user_treat, clicks_new_avg_user_control, \\\n",
    "    clicks_new_tot_user_treat, clicks_new_tot_user_control, \\\n",
    "    coldstart_rate_user_treat, coldstart_rate_user_control \\\n",
    "    = user_exp(opt = opt, is_new_index = is_new_index , \n",
    "               target_bid_price = target_bid_price, \n",
    "               coefficient_ctr_model = coefficient_ctr_model) \n",
    "        \n",
    "    rev_new_avg_t_t, rev_new_avg_c_c, rev_new_avg_t_c, rev_new_avg_c_t,  \\\n",
    "    rev_new_tot_t_t, rev_new_tot_c_c, rev_new_tot_t_c, rev_new_tot_c_t, \\\n",
    "    clicks_new_avg_t_t, clicks_new_avg_c_c, clicks_new_avg_t_c, clicks_new_avg_c_t, \\\n",
    "    clicks_new_tot_t_t, clicks_new_tot_c_c, clicks_new_tot_t_c, clicks_new_tot_c_t, \\\n",
    "    coldstart_rate_t_t, coldstart_rate_t_t, coldstart_rate_c_c, coldstart_rate_t_c, coldstart_rate_c_t \\\n",
    "    = twoside_exp(opt = opt, is_new_index = is_new_index,\n",
    "                  target_bid_price = target_bid_price,\n",
    "                  coefficient_ctr_model = coefficient_ctr_model) \n",
    "    # Append the results to the DataFrame\n",
    "    dict_row = {}\n",
    "    for j in columns:\n",
    "        dict_row[j] = eval(j)\n",
    "    results_df = results_df.append(dict_row, ignore_index=True)\n",
    "    \n",
    "    # Save the results to a CSV file after every loop\n",
    "    results_df.to_csv(\"simulation_results_seed{}.csv\".format(opt.seed), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['--seed'], dest='seed', nargs=None, const=None, default=13, type=<class 'int'>, choices=None, required=False, help='random seed', metavar=None)"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parser = argparse.ArgumentParser()\n",
    "#Default cold start parameters - the cold start value = reward*I(clicks>=target_click)\n",
    "parser.add_argument('--total_ads', type=int, default=3000, help='total number of ads for simulation')\n",
    "parser.add_argument('--new_ad_ratio', type=float, default=0.3, help='ratio of new ads')\n",
    "parser.add_argument('--total_pv', type=int, default=1000000, help='loaded total page views in one cold start horizon for simulation')\n",
    "parser.add_argument('--reward', type=float, default=2000000, help='reward of cold start')\n",
    "parser.add_argument('--delta',  type=float, default=600000, help='delta reward of treated group')\n",
    "parser.add_argument('--target_click', type=int, default=1000, help='target click')\n",
    "parser.add_argument('--alpha', type=float, default=1e-3, help='target click/total_pv')\n",
    "#If one wants to change total_ads, it is better to proportionally change total_pv to balance the two-side market\n",
    "\n",
    "#Others parameters\n",
    "parser.add_argument('--budget_limit', type=int, default=5e5, help='set an uniform budget limit on ads')\n",
    "parser.add_argument('--PID_cycle', type=int, default=100000, help='after every # pvs, update PID')\n",
    "\n",
    "#Parameters about experiments - sampling rate\n",
    "parser.add_argument('--ad_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "parser.add_argument('--pv_treatment_ratio', type=float, default=0.4, help='range from 0 to 0.5')\n",
    "\n",
    "#Parameters about model - learning rate\n",
    "parser.add_argument('--l_rate', type=float, default=0.004, help='learning rage')\n",
    "\n",
    "#Parameters about random - random seed\n",
    "parser.add_argument('--seed', type=int, default = 13, help='random seed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'total_ads': 3000, 'new_ad_ratio': 0.3, 'total_pv': 1000000, 'reward': 2000000, 'delta': 600000, 'target_click': 1000, 'alpha': 0.001, 'budget_limit': 500000.0, 'PID_cycle': 100000, 'ad_treatment_ratio': 0.4, 'pv_treatment_ratio': 0.4, 'l_rate': 0.004, 'seed': 13}\n"
     ]
    }
   ],
   "source": [
    "opt = parser.parse_args()\n",
    "np.save('my_config_seed{}.npy'.format(opt.seed), opt)\n",
    "print(opt.__dict__)\n",
    "np.random.seed(opt.seed)\n",
    "is_new_index, target_bid_price, coefficient_ctr_model = generate_ad(n = opt.total_ads, new_raito = opt.new_ad_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a432f7f45644a61bf874a6df4a58fef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Simulation progress:   0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfce497310764cdf9f5b254182436f70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.005100684766324836\n",
      "Total   Revenue in new: 4590616.2896923525\n",
      "Average clicks in new: 6.446444444444445e-05\n",
      "Total   clicks in new: 58018.0\n",
      "Cold Start Success Rate: 0.06111111111111111\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fe8a395accc41838e841392c50597d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.00665944924593625\n",
      "Total   Revenue in new: 5993504.321342625\n",
      "Average clicks in new: 7.032333333333333e-05\n",
      "Total   clicks in new: 63291.0\n",
      "Cold Start Success Rate: 0.06222222222222222\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e4e159bc1e44629ba64624dbf6e1e57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.009442039967656612\n",
      "Average Revenue in new of control group: 0.003052465093408084\n",
      "Total   Revenue in new of treat   group: 3399134.38835638\n",
      "Total   Revenue in new of control group: 1098887.4336269102\n",
      "Average clicks in new of treat   group: 0.00013111944444444445\n",
      "Average clicks in new of control group: 1.8477777777777777e-05\n",
      "Total   clicks in new of treat   group: 47203.0\n",
      "Total   clicks in new of control group: 6652.0\n",
      "Cold Start Success Rate of treat   group: 0.11666666666666667\n",
      "Cold Start Success Rate of control group: 0.016666666666666666\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "315ee85a957746b0b2b1dee90fa39cb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.011609700302117458\n",
      "Average Revenue in new of control group: 0.011507216499357624\n",
      "Total   Revenue in new of treat   group: 4179492.1087622847\n",
      "Total   Revenue in new of control group: 4142597.9397687446\n",
      "Average clicks in new of treat   group: 0.00014925833333333333\n",
      "Average clicks in new of control group: 0.00014766944444444445\n",
      "Total   clicks in new of treat   group: 53733.0\n",
      "Total   clicks in new of control group: 53161.0\n",
      "Cold Start Success Rate of treat   group: 0.13555555555555557\n",
      "Cold Start Success Rate of control group: 0.13444444444444445\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c720e2c80014703b8da9af54be4add9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.016650763709463366\n",
      "Average Revenue in new of c_c group: 0.006701620967923648\n",
      "Average Revenue in new of t_c group: 0.005707139053332892\n",
      "Average Revenue in new of c_t group: 0.016017073907493507\n",
      "Total   Revenue in new of t_t group: 2397709.974162725\n",
      "Total   Revenue in new of c_c group: 965033.4193810052\n",
      "Total   Revenue in new of t_c group: 821828.0236799364\n",
      "Total   Revenue in new of c_t group: 2306458.642679065\n",
      "Average clicks in new of t_t group: 0.0002112777777777778\n",
      "Average clicks in new of c_c group: 0.00010741666666666667\n",
      "Average clicks in new of t_c group: 8.901388888888888e-05\n",
      "Average clicks in new of c_t group: 0.00019911805555555554\n",
      "Total   clicks in new of t_t group: 30424.0\n",
      "Total   clicks in new of c_c group: 15468.0\n",
      "Total   clicks in new of t_c group: 12818.0\n",
      "Total   clicks in new of c_t group: 28673.0\n",
      "Cold Start Success Rate of t_t group: 0.18611111111111112\n",
      "Cold Start Success Rate of c_c group: 0.1\n",
      "Cold Start Success Rate of t_c group: 0.013888888888888888\n",
      "Cold Start Success Rate of c_t group: 0.17777777777777778\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c19b0f7d5815409f9e2929b567231304",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.004298086268549427\n",
      "Total   Revenue in new: 3868277.6416944843\n",
      "Average clicks in new: 5.575777777777778e-05\n",
      "Total   clicks in new: 50182.0\n",
      "Cold Start Success Rate: 0.04888888888888889\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e54f209459747c2bba719840f758c12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.0061609669602902956\n",
      "Total   Revenue in new: 5544870.264261266\n",
      "Average clicks in new: 7.255888888888889e-05\n",
      "Total   clicks in new: 65303.0\n",
      "Cold Start Success Rate: 0.06444444444444444\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e71adac50fd45efa20429a58960c4b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.010679422215443313\n",
      "Average Revenue in new of control group: 0.0009747023472197446\n",
      "Total   Revenue in new of treat   group: 3844591.997559593\n",
      "Total   Revenue in new of control group: 350892.84499910806\n",
      "Average clicks in new of treat   group: 0.00011145833333333334\n",
      "Average clicks in new of control group: 1.9719444444444445e-05\n",
      "Total   clicks in new of treat   group: 40125.0\n",
      "Total   clicks in new of control group: 7099.0\n",
      "Cold Start Success Rate of treat   group: 0.1\n",
      "Cold Start Success Rate of control group: 0.016666666666666666\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f8215b7affd433797e617e79493c054",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.013015895691770833\n",
      "Average Revenue in new of control group: 0.01292273106796446\n",
      "Total   Revenue in new of treat   group: 4685722.4490375\n",
      "Total   Revenue in new of control group: 4652183.184467206\n",
      "Average clicks in new of treat   group: 0.00014549166666666667\n",
      "Average clicks in new of control group: 0.00014396666666666668\n",
      "Total   clicks in new of treat   group: 52377.0\n",
      "Total   clicks in new of control group: 51828.0\n",
      "Cold Start Success Rate of treat   group: 0.13111111111111112\n",
      "Cold Start Success Rate of control group: 0.13111111111111112\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6acd722a157c45afbca051bfbb5ab9bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.01888118233299482\n",
      "Average Revenue in new of c_c group: 0.007759483439132772\n",
      "Average Revenue in new of t_c group: 0.006677268893685708\n",
      "Average Revenue in new of c_t group: 0.01787239423760513\n",
      "Total   Revenue in new of t_t group: 2718890.2559512537\n",
      "Total   Revenue in new of c_c group: 1117365.6152351191\n",
      "Total   Revenue in new of t_c group: 961526.7206907419\n",
      "Total   Revenue in new of c_t group: 2573624.770215139\n",
      "Average clicks in new of t_t group: 0.00022302083333333334\n",
      "Average clicks in new of c_c group: 9.56875e-05\n",
      "Average clicks in new of t_c group: 7.830555555555556e-05\n",
      "Average clicks in new of c_t group: 0.00021003472222222221\n",
      "Total   clicks in new of t_t group: 32115.0\n",
      "Total   clicks in new of c_c group: 13779.0\n",
      "Total   clicks in new of t_c group: 11276.0\n",
      "Total   clicks in new of c_t group: 30245.0\n",
      "Cold Start Success Rate of t_t group: 0.20277777777777778\n",
      "Cold Start Success Rate of c_c group: 0.08333333333333333\n",
      "Cold Start Success Rate of t_c group: 0.005555555555555556\n",
      "Cold Start Success Rate of c_t group: 0.18611111111111112\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b537e7bab9144649471f965880eda04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.004454483808672156\n",
      "Total   Revenue in new: 4009035.4278049404\n",
      "Average clicks in new: 6.175777777777778e-05\n",
      "Total   clicks in new: 55582.0\n",
      "Cold Start Success Rate: 0.05555555555555555\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b51f209ea66466ab25f974960b2acb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.006091771617228857\n",
      "Total   Revenue in new: 5482594.455505971\n",
      "Average clicks in new: 7.238444444444444e-05\n",
      "Total   clicks in new: 65146.0\n",
      "Cold Start Success Rate: 0.06444444444444444\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "731cd354244148c1b91e9ca37807f956",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.012081810297219851\n",
      "Average Revenue in new of control group: 0.0010890626558319398\n",
      "Total   Revenue in new of treat   group: 4349451.706999146\n",
      "Total   Revenue in new of control group: 392062.5560994983\n",
      "Average clicks in new of treat   group: 0.000138675\n",
      "Average clicks in new of control group: 6.780555555555556e-06\n",
      "Total   clicks in new of treat   group: 49923.0\n",
      "Total   clicks in new of control group: 2441.0\n",
      "Cold Start Success Rate of treat   group: 0.125\n",
      "Cold Start Success Rate of control group: 0.005555555555555556\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08e7e57d861b4cd2abb57edccef48c1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.014592836290940771\n",
      "Average Revenue in new of control group: 0.014353585691116336\n",
      "Total   Revenue in new of treat   group: 5253421.064738678\n",
      "Total   Revenue in new of control group: 5167290.848801881\n",
      "Average clicks in new of treat   group: 0.00015934166666666666\n",
      "Average clicks in new of control group: 0.00015611666666666668\n",
      "Total   clicks in new of treat   group: 57363.0\n",
      "Total   clicks in new of control group: 56202.0\n",
      "Cold Start Success Rate of treat   group: 0.14555555555555555\n",
      "Cold Start Success Rate of control group: 0.1411111111111111\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3a431e9f7d84e62bfde0e93ca4e0be2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.019390952934489884\n",
      "Average Revenue in new of c_c group: 0.008722491237304607\n",
      "Average Revenue in new of t_c group: 0.007269588360200497\n",
      "Average Revenue in new of c_t group: 0.018719959907169095\n",
      "Total   Revenue in new of t_t group: 2792297.222566543\n",
      "Total   Revenue in new of c_c group: 1256038.7381718634\n",
      "Total   Revenue in new of t_c group: 1046820.7238688715\n",
      "Total   Revenue in new of c_t group: 2695674.2266323497\n",
      "Average clicks in new of t_t group: 0.00022003472222222221\n",
      "Average clicks in new of c_c group: 0.00010829861111111111\n",
      "Average clicks in new of t_c group: 9.0125e-05\n",
      "Average clicks in new of c_t group: 0.00020972222222222223\n",
      "Total   clicks in new of t_t group: 31685.0\n",
      "Total   clicks in new of c_c group: 15595.0\n",
      "Total   clicks in new of t_c group: 12978.0\n",
      "Total   clicks in new of c_t group: 30200.0\n",
      "Cold Start Success Rate of t_t group: 0.20555555555555555\n",
      "Cold Start Success Rate of c_c group: 0.1\n",
      "Cold Start Success Rate of t_c group: 0.005555555555555556\n",
      "Cold Start Success Rate of c_t group: 0.19444444444444445\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0467c67f02f1455fae859fdd1dad2edc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.0052069522394512435\n",
      "Total   Revenue in new: 4686257.0155061195\n",
      "Average clicks in new: 5.953555555555556e-05\n",
      "Total   clicks in new: 53582.0\n",
      "Cold Start Success Rate: 0.052222222222222225\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eecdd0eda1a6497eae56964dfbc96274",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.005716897516601951\n",
      "Total   Revenue in new: 5145207.764941756\n",
      "Average clicks in new: 6.828777777777778e-05\n",
      "Total   clicks in new: 61459.0\n",
      "Cold Start Success Rate: 0.057777777777777775\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c354db745e3144119ea799d87844b140",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.009713794263010488\n",
      "Average Revenue in new of control group: 0.0029217879905981256\n",
      "Total   Revenue in new of treat   group: 3496965.9346837755\n",
      "Total   Revenue in new of control group: 1051843.6766153253\n",
      "Average clicks in new of treat   group: 0.00011217222222222223\n",
      "Average clicks in new of control group: 3.1011111111111114e-05\n",
      "Total   clicks in new of treat   group: 40382.0\n",
      "Total   clicks in new of control group: 11164.0\n",
      "Cold Start Success Rate of treat   group: 0.1\n",
      "Cold Start Success Rate of control group: 0.022222222222222223\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86287e4fcede425a8c18fcbee7df67ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.01396355590830854\n",
      "Average Revenue in new of control group: 0.013785416435491615\n",
      "Total   Revenue in new of treat   group: 5026880.1269910745\n",
      "Total   Revenue in new of control group: 4962749.916776981\n",
      "Average clicks in new of treat   group: 0.00014571944444444446\n",
      "Average clicks in new of control group: 0.00014375\n",
      "Total   clicks in new of treat   group: 52459.0\n",
      "Total   clicks in new of control group: 51750.0\n",
      "Cold Start Success Rate of treat   group: 0.13444444444444445\n",
      "Cold Start Success Rate of control group: 0.13111111111111112\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "545cba9088304a2abf0327b2a2ba910d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.019939482067832873\n",
      "Average Revenue in new of c_c group: 0.008198736016757465\n",
      "Average Revenue in new of t_c group: 0.007261931634617444\n",
      "Average Revenue in new of c_t group: 0.019007664459974833\n",
      "Total   Revenue in new of t_t group: 2871285.4177679336\n",
      "Total   Revenue in new of c_c group: 1180617.986413075\n",
      "Total   Revenue in new of t_c group: 1045718.1553849119\n",
      "Total   Revenue in new of c_t group: 2737103.6822363758\n",
      "Average clicks in new of t_t group: 0.00019175694444444446\n",
      "Average clicks in new of c_c group: 9.924305555555556e-05\n",
      "Average clicks in new of t_c group: 8.293055555555556e-05\n",
      "Average clicks in new of c_t group: 0.000182375\n",
      "Total   clicks in new of t_t group: 27613.0\n",
      "Total   clicks in new of c_c group: 14291.0\n",
      "Total   clicks in new of t_c group: 11942.0\n",
      "Total   clicks in new of c_t group: 26262.0\n",
      "Cold Start Success Rate of t_t group: 0.175\n",
      "Cold Start Success Rate of c_c group: 0.09166666666666666\n",
      "Cold Start Success Rate of t_c group: 0.011111111111111112\n",
      "Cold Start Success Rate of c_t group: 0.16944444444444445\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "223837dda3644c62a230f52ea52ceb26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.00505316596518025\n",
      "Total   Revenue in new: 4547849.368662225\n",
      "Average clicks in new: 6.0948888888888886e-05\n",
      "Total   clicks in new: 54854.0\n",
      "Cold Start Success Rate: 0.056666666666666664\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40c2ecaefd7d4782b6c373ccb938c047",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.006406247605402964\n",
      "Total   Revenue in new: 5765622.844862668\n",
      "Average clicks in new: 7.242333333333333e-05\n",
      "Total   clicks in new: 65181.0\n",
      "Cold Start Success Rate: 0.06555555555555556\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbaff4d5f98441e3b3c3cc4c39ecb03b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.010736916514203238\n",
      "Average Revenue in new of control group: 0.0015086454184543112\n",
      "Total   Revenue in new of treat   group: 3865289.945113166\n",
      "Total   Revenue in new of control group: 543112.350643552\n",
      "Average clicks in new of treat   group: 0.00013192222222222223\n",
      "Average clicks in new of control group: 1.495e-05\n",
      "Total   clicks in new of treat   group: 47492.0\n",
      "Total   clicks in new of control group: 5382.0\n",
      "Cold Start Success Rate of treat   group: 0.1111111111111111\n",
      "Cold Start Success Rate of control group: 0.013888888888888888\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "804a94b8a0d543bf813b066dd5dffce2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.01293115680943948\n",
      "Average Revenue in new of control group: 0.012891095231803604\n",
      "Total   Revenue in new of treat   group: 4655216.451398212\n",
      "Total   Revenue in new of control group: 4640794.283449298\n",
      "Average clicks in new of treat   group: 0.00015010277777777778\n",
      "Average clicks in new of control group: 0.00014856944444444444\n",
      "Total   clicks in new of treat   group: 54037.0\n",
      "Total   clicks in new of control group: 53485.0\n",
      "Cold Start Success Rate of treat   group: 0.13777777777777778\n",
      "Cold Start Success Rate of control group: 0.13333333333333333\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbf48d81ed904ccc9743a4257ca56bca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.01795859174755716\n",
      "Average Revenue in new of c_c group: 0.0075112832867082245\n",
      "Average Revenue in new of t_c group: 0.006789951186011767\n",
      "Average Revenue in new of c_t group: 0.016770066217418805\n",
      "Total   Revenue in new of t_t group: 2586037.2116482314\n",
      "Total   Revenue in new of c_c group: 1081624.7932859843\n",
      "Total   Revenue in new of t_c group: 977752.9707856945\n",
      "Total   Revenue in new of c_t group: 2414889.535308308\n",
      "Average clicks in new of t_t group: 0.00021763888888888888\n",
      "Average clicks in new of c_c group: 9.029166666666666e-05\n",
      "Average clicks in new of t_c group: 7.652777777777778e-05\n",
      "Average clicks in new of c_t group: 0.00020530555555555555\n",
      "Total   clicks in new of t_t group: 31340.0\n",
      "Total   clicks in new of c_c group: 13002.0\n",
      "Total   clicks in new of t_c group: 11020.0\n",
      "Total   clicks in new of c_t group: 29564.0\n",
      "Cold Start Success Rate of t_t group: 0.19444444444444445\n",
      "Cold Start Success Rate of c_c group: 0.08611111111111111\n",
      "Cold Start Success Rate of t_c group: 0.008333333333333333\n",
      "Cold Start Success Rate of c_t group: 0.18611111111111112\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef78d51a77f8421894a38e7dc1035cb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.004574030669960611\n",
      "Total   Revenue in new: 4116627.60296455\n",
      "Average clicks in new: 5.8555555555555557e-05\n",
      "Total   clicks in new: 52700.0\n",
      "Cold Start Success Rate: 0.05444444444444444\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2b3b8d8db224a449122ae67c67260a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.00689104997349698\n",
      "Total   Revenue in new: 6201944.976147282\n",
      "Average clicks in new: 7.306666666666666e-05\n",
      "Total   clicks in new: 65760.0\n",
      "Cold Start Success Rate: 0.06444444444444444\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ff9a76e1e8c4f2c82f568f1dc9849f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.011049764568075127\n",
      "Average Revenue in new of control group: 2.7550193140829278e-06\n",
      "Total   Revenue in new of treat   group: 3977915.2445070455\n",
      "Total   Revenue in new of control group: 991.806953069854\n",
      "Average clicks in new of treat   group: 0.00013802222222222222\n",
      "Average clicks in new of control group: 1.9444444444444445e-08\n",
      "Total   clicks in new of treat   group: 49688.0\n",
      "Total   clicks in new of control group: 7.0\n",
      "Cold Start Success Rate of treat   group: 0.13055555555555556\n",
      "Cold Start Success Rate of control group: 0.0\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6b107d3078340a4ba19be02ed4811bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.012293586447590966\n",
      "Average Revenue in new of control group: 0.012200094822916317\n",
      "Total   Revenue in new of treat   group: 4425691.121132748\n",
      "Total   Revenue in new of control group: 4392034.136249874\n",
      "Average clicks in new of treat   group: 0.00014163333333333332\n",
      "Average clicks in new of control group: 0.00013991666666666666\n",
      "Total   clicks in new of treat   group: 50988.0\n",
      "Total   clicks in new of control group: 50370.0\n",
      "Cold Start Success Rate of treat   group: 0.1288888888888889\n",
      "Cold Start Success Rate of control group: 0.12777777777777777\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "487018e7f17f4ec59d6d0fa80686d543",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.013196043823410044\n",
      "Average Revenue in new of c_c group: 0.010237517254177264\n",
      "Average Revenue in new of t_c group: 0.009306018041960412\n",
      "Average Revenue in new of c_t group: 0.012534647055344506\n",
      "Total   Revenue in new of t_t group: 1900230.3105710463\n",
      "Total   Revenue in new of c_c group: 1474202.484601526\n",
      "Total   Revenue in new of t_c group: 1340066.5980422993\n",
      "Total   Revenue in new of c_t group: 1804989.1759696088\n",
      "Average clicks in new of t_t group: 0.00017265972222222221\n",
      "Average clicks in new of c_c group: 0.00012540972222222223\n",
      "Average clicks in new of t_c group: 0.00010940277777777778\n",
      "Average clicks in new of c_t group: 0.0001603888888888889\n",
      "Total   clicks in new of t_t group: 24863.0\n",
      "Total   clicks in new of c_c group: 18059.0\n",
      "Total   clicks in new of t_c group: 15754.0\n",
      "Total   clicks in new of c_t group: 23096.0\n",
      "Cold Start Success Rate of t_t group: 0.15833333333333333\n",
      "Cold Start Success Rate of c_c group: 0.11666666666666667\n",
      "Cold Start Success Rate of t_c group: 0.016666666666666666\n",
      "Cold Start Success Rate of c_t group: 0.14722222222222223\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1eef957c2f4747c4a558618f928e1403",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.00633927347338574\n",
      "Total   Revenue in new: 5705346.126047166\n",
      "Average clicks in new: 6.340111111111111e-05\n",
      "Total   clicks in new: 57061.0\n",
      "Cold Start Success Rate: 0.058888888888888886\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "136f8e232fe2408ca3b0157789c8fda7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.008143898089466717\n",
      "Total   Revenue in new: 7329508.280520045\n",
      "Average clicks in new: 7.337888888888889e-05\n",
      "Total   clicks in new: 66041.0\n",
      "Cold Start Success Rate: 0.06444444444444444\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5755982a513540808e5bde133fe37e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.01393556488191278\n",
      "Average Revenue in new of control group: 0.0022568714394433182\n",
      "Total   Revenue in new of treat   group: 5016803.357488601\n",
      "Total   Revenue in new of control group: 812473.7181995946\n",
      "Average clicks in new of treat   group: 0.00014216666666666666\n",
      "Average clicks in new of control group: 1.9494444444444443e-05\n",
      "Total   clicks in new of treat   group: 51180.0\n",
      "Total   clicks in new of control group: 7018.0\n",
      "Cold Start Success Rate of treat   group: 0.12777777777777777\n",
      "Cold Start Success Rate of control group: 0.019444444444444445\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc21e17509c845cd9be9b8a06265fb08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.015534790901064632\n",
      "Average Revenue in new of control group: 0.015261132532558315\n",
      "Total   Revenue in new of treat   group: 5592524.724383268\n",
      "Total   Revenue in new of control group: 5494007.711720994\n",
      "Average clicks in new of treat   group: 0.0001539138888888889\n",
      "Average clicks in new of control group: 0.00015090555555555556\n",
      "Total   clicks in new of treat   group: 55409.0\n",
      "Total   clicks in new of control group: 54326.0\n",
      "Cold Start Success Rate of treat   group: 0.13666666666666666\n",
      "Cold Start Success Rate of control group: 0.13444444444444445\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a979d5989714f4d9597b319f500dd15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.023293127461867246\n",
      "Average Revenue in new of c_c group: 0.0074477892973563945\n",
      "Average Revenue in new of t_c group: 0.0060468032538139975\n",
      "Average Revenue in new of c_t group: 0.0220711325430765\n",
      "Total   Revenue in new of t_t group: 3354210.3545088833\n",
      "Total   Revenue in new of c_c group: 1072481.6588193208\n",
      "Total   Revenue in new of t_c group: 870739.6685492157\n",
      "Total   Revenue in new of c_t group: 3178243.0862030163\n",
      "Average clicks in new of t_t group: 0.0002355902777777778\n",
      "Average clicks in new of c_c group: 8.933333333333333e-05\n",
      "Average clicks in new of t_c group: 7.086111111111111e-05\n",
      "Average clicks in new of c_t group: 0.00022117361111111112\n",
      "Total   clicks in new of t_t group: 33925.0\n",
      "Total   clicks in new of c_c group: 12864.0\n",
      "Total   clicks in new of t_c group: 10204.0\n",
      "Total   clicks in new of c_t group: 31849.0\n",
      "Cold Start Success Rate of t_t group: 0.20555555555555555\n",
      "Cold Start Success Rate of c_c group: 0.07222222222222222\n",
      "Cold Start Success Rate of t_c group: 0.002777777777777778\n",
      "Cold Start Success Rate of c_t group: 0.19166666666666668\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89b5e9ded0ae4ba384d3ea0287b0f3b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.006182349780180228\n",
      "Total   Revenue in new: 5564114.802162205\n",
      "Average clicks in new: 6.0293333333333334e-05\n",
      "Total   clicks in new: 54264.0\n",
      "Cold Start Success Rate: 0.051111111111111114\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfac7234bde7435590e314a2d45c7977",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.007718135942713143\n",
      "Total   Revenue in new: 6946322.348441829\n",
      "Average clicks in new: 7.608222222222223e-05\n",
      "Total   clicks in new: 68474.0\n",
      "Cold Start Success Rate: 0.07\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f06b7a1381044b409442c9ffcf9929f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.012599263238872373\n",
      "Average Revenue in new of control group: 0.003008208533594663\n",
      "Total   Revenue in new of treat   group: 4535734.765994054\n",
      "Total   Revenue in new of control group: 1082955.0720940786\n",
      "Average clicks in new of treat   group: 0.000136075\n",
      "Average clicks in new of control group: 1.4325e-05\n",
      "Total   clicks in new of treat   group: 48987.0\n",
      "Total   clicks in new of control group: 5157.0\n",
      "Cold Start Success Rate of treat   group: 0.11944444444444445\n",
      "Cold Start Success Rate of control group: 0.011111111111111112\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f87a049ed9b4ae2a8a8aeac6813001f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.013617649536716297\n",
      "Average Revenue in new of control group: 0.013473447464418442\n",
      "Total   Revenue in new of treat   group: 4902353.833217867\n",
      "Total   Revenue in new of control group: 4850441.087190639\n",
      "Average clicks in new of treat   group: 0.00014648333333333333\n",
      "Average clicks in new of control group: 0.00014487777777777777\n",
      "Total   clicks in new of treat   group: 52734.0\n",
      "Total   clicks in new of control group: 52156.0\n",
      "Cold Start Success Rate of treat   group: 0.13555555555555557\n",
      "Cold Start Success Rate of control group: 0.1322222222222222\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18ff8d38d04641a6b4ebd5ab76d0f971",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.020359441777850058\n",
      "Average Revenue in new of c_c group: 0.007408086391390235\n",
      "Average Revenue in new of t_c group: 0.0062976719879429535\n",
      "Average Revenue in new of c_t group: 0.019718901192836837\n",
      "Total   Revenue in new of t_t group: 2931759.6160104084\n",
      "Total   Revenue in new of c_c group: 1066764.4403601938\n",
      "Total   Revenue in new of t_c group: 906864.7662637853\n",
      "Total   Revenue in new of c_t group: 2839521.7717685048\n",
      "Average clicks in new of t_t group: 0.00022058333333333335\n",
      "Average clicks in new of c_c group: 9.946527777777777e-05\n",
      "Average clicks in new of t_c group: 8.094444444444444e-05\n",
      "Average clicks in new of c_t group: 0.0002083888888888889\n",
      "Total   clicks in new of t_t group: 31764.0\n",
      "Total   clicks in new of c_c group: 14323.0\n",
      "Total   clicks in new of t_c group: 11656.0\n",
      "Total   clicks in new of c_t group: 30008.0\n",
      "Cold Start Success Rate of t_t group: 0.19444444444444445\n",
      "Cold Start Success Rate of c_c group: 0.09166666666666666\n",
      "Cold Start Success Rate of t_c group: 0.005555555555555556\n",
      "Cold Start Success Rate of c_t group: 0.18611111111111112\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dce37995cb14ed6bdd7b8acd7c0a936",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.006258738937852581\n",
      "Total   Revenue in new: 5632865.044067323\n",
      "Average clicks in new: 5.995555555555556e-05\n",
      "Total   clicks in new: 53960.0\n",
      "Cold Start Success Rate: 0.05444444444444444\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "724fce7c61ab4f16b5f60d164786fd6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.008805661836501963\n",
      "Total   Revenue in new: 7925095.652851767\n",
      "Average clicks in new: 7.519222222222223e-05\n",
      "Total   clicks in new: 67673.0\n",
      "Cold Start Success Rate: 0.06666666666666667\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c58320060f74478bc682182bc05ad86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.012758674944820953\n",
      "Average Revenue in new of control group: 0.006672663868565058\n",
      "Total   Revenue in new of treat   group: 4593122.980135543\n",
      "Total   Revenue in new of control group: 2402158.992683421\n",
      "Average clicks in new of treat   group: 0.000134\n",
      "Average clicks in new of control group: 2.613888888888889e-05\n",
      "Total   clicks in new of treat   group: 48240.0\n",
      "Total   clicks in new of control group: 9410.0\n",
      "Cold Start Success Rate of treat   group: 0.11666666666666667\n",
      "Cold Start Success Rate of control group: 0.019444444444444445\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1566de8287ea4e5d95eda41ecb8edb1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.013474374500872287\n",
      "Average Revenue in new of control group: 0.013544445794725702\n",
      "Total   Revenue in new of treat   group: 4850774.820314024\n",
      "Total   Revenue in new of control group: 4876000.486101253\n",
      "Average clicks in new of treat   group: 0.00014125555555555557\n",
      "Average clicks in new of control group: 0.00014068888888888888\n",
      "Total   clicks in new of treat   group: 50852.0\n",
      "Total   clicks in new of control group: 50648.0\n",
      "Cold Start Success Rate of treat   group: 0.12444444444444444\n",
      "Cold Start Success Rate of control group: 0.12\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33edb1eac69c4c64b79cf6f1f2ecff37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.02540739653524761\n",
      "Average Revenue in new of c_c group: 0.003435631834269198\n",
      "Average Revenue in new of t_c group: 0.0027854103312740432\n",
      "Average Revenue in new of c_t group: 0.02446902328609517\n",
      "Total   Revenue in new of t_t group: 3658665.101075656\n",
      "Total   Revenue in new of c_c group: 494730.9841347645\n",
      "Total   Revenue in new of t_c group: 401099.0877034622\n",
      "Total   Revenue in new of c_t group: 3523539.3531977045\n",
      "Average clicks in new of t_t group: 0.00022903472222222222\n",
      "Average clicks in new of c_c group: 7.222222222222222e-05\n",
      "Average clicks in new of t_c group: 5.7895833333333336e-05\n",
      "Average clicks in new of c_t group: 0.00021677083333333333\n",
      "Total   clicks in new of t_t group: 32981.0\n",
      "Total   clicks in new of c_c group: 10400.0\n",
      "Total   clicks in new of t_c group: 8337.0\n",
      "Total   clicks in new of c_t group: 31215.0\n",
      "Cold Start Success Rate of t_t group: 0.20277777777777778\n",
      "Cold Start Success Rate of c_c group: 0.06666666666666667\n",
      "Cold Start Success Rate of t_c group: 0.005555555555555556\n",
      "Cold Start Success Rate of c_t group: 0.18888888888888888\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba14ddb3938a473388915a7bd3e15024",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All control experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.004903444191845703\n",
      "Total   Revenue in new: 4413099.772661133\n",
      "Average clicks in new: 5.835111111111111e-05\n",
      "Total   clicks in new: 52516.0\n",
      "Cold Start Success Rate: 0.05\n",
      "================================================\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "695e185a2285419db5a391b8f0683523",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All treat experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new: 0.0053544120003889825\n",
      "Total   Revenue in new: 4818970.800350084\n",
      "Average clicks in new: 6.504777777777777e-05\n",
      "Total   clicks in new: 58543.0\n",
      "Cold Start Success Rate: 0.058888888888888886\n",
      "================================================\n",
      "Ad-side experiment\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "221fc3a4bff342fa9f2e236a71f7bd00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ad side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.009369943607957986\n",
      "Average Revenue in new of control group: 0.0015598768871605882\n",
      "Total   Revenue in new of treat   group: 3373179.6988648754\n",
      "Total   Revenue in new of control group: 561555.6793778117\n",
      "Average clicks in new of treat   group: 0.00011841388888888889\n",
      "Average clicks in new of control group: 1.606111111111111e-05\n",
      "Total   clicks in new of treat   group: 42629.0\n",
      "Total   clicks in new of control group: 5782.0\n",
      "Cold Start Success Rate of treat   group: 0.10277777777777777\n",
      "Cold Start Success Rate of control group: 0.013888888888888888\n",
      "================================================\n",
      "User-side experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0110cca0bf574de287a50a93cb4af45d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "user side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of treat   group: 0.012624428813629619\n",
      "Average Revenue in new of control group: 0.012530144958728702\n",
      "Total   Revenue in new of treat   group: 4544794.3729066625\n",
      "Total   Revenue in new of control group: 4510852.185142333\n",
      "Average clicks in new of treat   group: 0.00014231944444444445\n",
      "Average clicks in new of control group: 0.0001410722222222222\n",
      "Total   clicks in new of treat   group: 51235.0\n",
      "Total   clicks in new of control group: 50786.0\n",
      "Cold Start Success Rate of treat   group: 0.1288888888888889\n",
      "Cold Start Success Rate of control group: 0.12777777777777777\n",
      "================================================\n",
      "Two side randomized experiment\n",
      "# of users in treatment condition 400000\n",
      "# of users in control condition 400000\n",
      "# of new ads in treatment condition 360\n",
      "# of new ads in control condition 360\n",
      "# of t_t pairs 144000000\n",
      "# of t_c pairs 144000000\n",
      "# of c_t pairs 144000000\n",
      "# of c_c pairs 144000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "910f5e333fe046b68d385bf43cd86439",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "two side experiment:   0%|          | 0/1000000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Average Revenue in new of t_t group: 0.013622302499042106\n",
      "Average Revenue in new of c_c group: 0.011124643602561456\n",
      "Average Revenue in new of t_c group: 0.00967947709093798\n",
      "Average Revenue in new of c_t group: 0.0129599953680932\n",
      "Total   Revenue in new of t_t group: 1961611.5598620633\n",
      "Total   Revenue in new of c_c group: 1601948.6787688497\n",
      "Total   Revenue in new of t_c group: 1393844.701095069\n",
      "Total   Revenue in new of c_t group: 1866239.3330054209\n",
      "Average clicks in new of t_t group: 0.00019340972222222223\n",
      "Average clicks in new of c_c group: 9.0125e-05\n",
      "Average clicks in new of t_c group: 7.655555555555555e-05\n",
      "Average clicks in new of c_t group: 0.00018197222222222223\n",
      "Total   clicks in new of t_t group: 27851.0\n",
      "Total   clicks in new of c_c group: 12978.0\n",
      "Total   clicks in new of t_c group: 11024.0\n",
      "Total   clicks in new of c_t group: 26204.0\n",
      "Cold Start Success Rate of t_t group: 0.18055555555555555\n",
      "Cold Start Success Rate of c_c group: 0.07777777777777778\n",
      "Cold Start Success Rate of t_c group: 0.011111111111111112\n",
      "Cold Start Success Rate of c_t group: 0.16666666666666666\n",
      "================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86152\\AppData\\Local\\Temp\\ipykernel_8572\\27542452.py:58: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  results_df = results_df.append(dict_row, ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "# Initialize an empty DataFrame to store the results\n",
    "metrecs = [\"rev_new_avg\",\"rev_new_tot\",\"clicks_new_avg\",'clicks_new_tot','coldstart_rate']\n",
    "exp = ['_all_treat', '_all_control', \n",
    "       '_ad_treat', '_ad_control', \n",
    "       '_user_treat', '_user_control', \n",
    "       '_t_t', '_c_c', '_t_c', '_c_t']\n",
    "columns = [i+j for i in metrecs for j in exp ]\n",
    "results_df = pd.DataFrame(columns = columns)\n",
    "\n",
    "# Run the simulation loop 10 times\n",
    "for i in tqdm(range(10), desc=\"Simulation progress\"):\n",
    "    opt = parser.parse_args()\n",
    "    is_new_index, target_bid_price, coefficient_ctr_model \\\n",
    "    = generate_ad(n = opt.total_ads, new_raito = opt.new_ad_ratio)\n",
    "\n",
    "    rev_new_avg_all_control, rev_new_tot_all_control, \\\n",
    "    clicks_new_avg_all_control, clicks_new_tot_all_control, \\\n",
    "    coldstart_rate_all_control = all_control(opt = opt, is_new_index = is_new_index , \n",
    "                                target_bid_price = target_bid_price, \n",
    "                                coefficient_ctr_model = coefficient_ctr_model)\n",
    "    \n",
    "    rev_new_avg_all_treat,   rev_new_tot_all_treat,   \\\n",
    "    clicks_new_avg_all_treat,   clicks_new_tot_all_treat,   \\\n",
    "    coldstart_rate_all_treat  = all_treat(opt = opt, is_new_index = is_new_index , \n",
    "                                target_bid_price = target_bid_price, \n",
    "                                coefficient_ctr_model = coefficient_ctr_model)\n",
    "\n",
    "    rev_new_avg_ad_treat, rev_new_avg_ad_control, \\\n",
    "    rev_new_tot_ad_treat, rev_new_tot_ad_control, \\\n",
    "    clicks_new_avg_ad_treat, clicks_new_avg_ad_control, \\\n",
    "    clicks_new_tot_ad_treat, clicks_new_tot_ad_control, \\\n",
    "    coldstart_rate_ad_treat, coldstart_rate_ad_control \\\n",
    "    = ad_exp(opt = opt, is_new_index = is_new_index , \n",
    "             target_bid_price = target_bid_price, \n",
    "             coefficient_ctr_model = coefficient_ctr_model) \n",
    "\n",
    "    rev_new_avg_user_treat, rev_new_avg_user_control, \\\n",
    "    rev_new_tot_user_treat, rev_new_tot_user_control, \\\n",
    "    clicks_new_avg_user_treat, clicks_new_avg_user_control, \\\n",
    "    clicks_new_tot_user_treat, clicks_new_tot_user_control, \\\n",
    "    coldstart_rate_user_treat, coldstart_rate_user_control \\\n",
    "    = user_exp(opt = opt, is_new_index = is_new_index , \n",
    "               target_bid_price = target_bid_price, \n",
    "               coefficient_ctr_model = coefficient_ctr_model) \n",
    "        \n",
    "    rev_new_avg_t_t, rev_new_avg_c_c, rev_new_avg_t_c, rev_new_avg_c_t,  \\\n",
    "    rev_new_tot_t_t, rev_new_tot_c_c, rev_new_tot_t_c, rev_new_tot_c_t, \\\n",
    "    clicks_new_avg_t_t, clicks_new_avg_c_c, clicks_new_avg_t_c, clicks_new_avg_c_t, \\\n",
    "    clicks_new_tot_t_t, clicks_new_tot_c_c, clicks_new_tot_t_c, clicks_new_tot_c_t, \\\n",
    "    coldstart_rate_t_t, coldstart_rate_t_t, coldstart_rate_c_c, coldstart_rate_t_c, coldstart_rate_c_t \\\n",
    "    = twoside_exp(opt = opt, is_new_index = is_new_index,\n",
    "                  target_bid_price = target_bid_price,\n",
    "                  coefficient_ctr_model = coefficient_ctr_model) \n",
    "    # Append the results to the DataFrame\n",
    "    dict_row = {}\n",
    "    for j in columns:\n",
    "        dict_row[j] = eval(j)\n",
    "    results_df = results_df.append(dict_row, ignore_index=True)\n",
    "    \n",
    "    # Save the results to a CSV file after every loop\n",
    "    results_df.to_csv(\"simulation_results_seed{}.csv\".format(opt.seed), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}