How to: Set up Arrow AB Test

Setting up Arrow AB Test on Shopify store

Set up AB testing of Arrow on your online store for the data-driven approach of adopting Arrow to see the impact of conversion uplift. Recommendation: A general setup of 50:50 split. 50% of traffic experience the existing checkout experience that directs shoppers through the original checkout experience. The other 50% experience Arrow where the checkout buttons embedded with Arrow direct shoppers to the Arrow checkout experience.

1. Initialize Mixpanel and store mixpanel.get_distinct_id value as cookie

Mixpanel is an analytics tool that tracks events and provide ease in maintaining identity across events tracked, so that user's journey across the platform can be stringed together and analysed. We initialize Mixpanel on your store and obtain the distinct_id that Mixpanel generates for the user. This same ID can be passed on to Arrow Checkout and we can attribute the subsequent events to the same user.

Paste the below code at the section:

<script type="text/javascript">
(function(f,b){if(!b.__SV){var e,g,i,h;window.mixpanel=b;b._i=[];b.init=function(e,f,c){function g(a,d){var b=d.split(".");2==b.length&&(a=a[b[0]],d=b[1]);a[d]=function(){a.push([d].concat(Array.prototype.slice.call(arguments,0)))}}var a=b;"undefined"!==typeof c?a=b[c]=[]:c="mixpanel";a.people=a.people||[];a.toString=function(a){var d="mixpanel";"mixpanel"!==c&&(d+="."+c);a||(d+=" (stub)");return d};a.people.toString=function(){return a.toString(1)+".people (stub)"};i="disable time_event track track_pageview track_links track_forms track_with_groups add_group set_group remove_group register register_once alias unregister identify name_tag set_config reset opt_in_tracking opt_out_tracking has_opted_in_tracking has_opted_out_tracking clear_opt_in_out_tracking start_batch_senders people.set people.set_once people.unset people.increment people.append people.union people.track_charge people.clear_charges people.delete_user people.remove".split(" ");
for(h=0;h<i.length;h++)g(a,i[h]);var j="set set_once union unset remove delete".split(" ");a.get_group=function(){function b(c){d[c]=function(){call2_args=arguments;call2=[c].concat(Array.prototype.slice.call(call2_args,0));a.push([e,call2])}}for(var d={},e=["get_group"].concat(Array.prototype.slice.call(arguments,0)),c=0;c<j.length;c++)b(j[c]);return d};b._i.push([e,f,c])};b.__SV=1.2;e=f.createElement("script");e.type="text/javascript";e.async=!0;e.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?
MIXPANEL_CUSTOM_LIB_URL:"file:"===f.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";g=f.getElementsByTagName("script")[0];g.parentNode.insertBefore(e,g)}})(document,window.mixpanel||[]);

// initiate mixpanel function and store distinct id as cookie
  mixpanel.init("9ba527872ab64cef0d90d69567730db1", {
    loaded: function(mixpanel) {
        distinct_id = mixpanel.get_distinct_id();
        setCookie('mx_distinct_id', distinct_id, 90);
    }
  });

</script>

2. Generate AB testing ID

Note: It is important for each unique user to be assigned to an experiment variant, and for his/her identity to be persisted in that same variant. For e.g., once a user has been assigned to 'Group A', he should be in the same 'Group A' experience.

We will generate an unique ID for the user to store on his/her browser cookie. This ID will be used to obtain an assigned variant, and to identify this user on a given browser (device). As long as we can retrieve a stored ID on the cookie, we can always fall back to using this generated value.

<script>
  function uuidv4() {
   return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
     (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
   );
  }
  
  function setCookie(cname, cvalue, exdays) {
    const d = new Date();
    d.setTime(d.getTime() + (exdays * 24 * 60 * 60 * 1000));
    let expires = "expires="+d.toUTCString();
    document.cookie = cname + "=" + cvalue + ";" + expires + ";path=/";
 }

 function getCookie(cname) {
    let name = cname + "=";
    let ca = document.cookie.split(';');
    for(let i = 0; i < ca.length; i++) {
     let c = ca[i];
     while (c.charAt(0) == ' ') {
       c = c.substring(1);
     }
     if (c.indexOf(name) == 0) {
       return c.substring(name.length, c.length);
     }
    }
     return "";
  }
  
  var ab_user_id = getCookie('ab_user_id');
  var mx_distinct_id = getCookie('mx_distinct_id');
  
  if(ab_user_id == null || ab_user_id == "" || ab_user_id == undefined){
    ab_user_id = uuidv4();
    setCookie("ab_user_id", ab_user_id, 90);
  }
</script>

3. Call the AB variant assignment endpoint

Once the Mixpanel and AB testing IDs have been prepared, we call the AB variant assignment endpoint to obtain a variant assigned for this user.

The endpoint:

We have to parse the response returning from the endpoint. Should the response informs show_arrow as false, we have to run a custom function to hide the Arrow button or display the standard checkout button. 

<script>
$.ajax({
    type: 'post',
    url: 'https://fly.witharrow.co/api/arrow/arrow_abtest',
    dataType: 'json',
    data: {
      client_uuid : ab_user_id,
      merchant_id : 244 ,
    },
    cache: false,
    success: function(returndata) {
      console.log("response");
      console.log(returndata);
      if(returndata['data']['show_arrow'] === false){
         $("#arrow-checkout-block-cart").hide();
         changeCartPageCheckoutButtonText();
      }

      mxAfterAjax(returndata, 'CartPageReach', window.location.href, mx_distinct_id)
    },
    error: function() { 
      console.error('Failed to process ajax !');
    }

  });
</script>

4. Fire the Mixpanel event bringing along with it the 'Variant ID'

Once the variant assignment has been obtained and the corresponding experience has been delivered, we would need to fire the event to Mixpanel for tracking purposes. The event that we fire is usually either CartPageReach or ProductPageReach, since generally we are interested to track the conversion difference starting from the customer landing on the product page, through to completing payment.

It is important to pass along the Variant ID, given that it will allow us to easily split the funnel view based on 'Group A' vs 'Group B' and make performance comparison on Mixpanel. Remember to also pass along the mx_distinct_id that should have been obtained from cookie.

Add the below function to section to easily call from either cart/product pages:

function mxAfterAjax(data, eventName, location, mx_distinct_id){
  console.log(data, mx_distinct_id)
  var interval = setInterval(function () {
    if (mx_distinct_id.length > 0) {
      mixpanel.track(eventName, {
        'MerchantID': 6,
        'Origin' : 'E-Store',
        'VariantId' : data.data.variant_id,
        'PageUrl': location
      }); 
      clearInterval(interval);
    }
  }, 300);
}

From product page, it can be used like this:

.
.
.    
success: function(returndata) {
      if(returndata['data']['show_arrow'] === false){
         $("#arrow-checkout-block").hide();
      }
//    mixpanel tracking after get variant_id
      mxAfterAjax(returndata, 'ProductPageReach', window.location.href, mx_distinct_id)
    },

You have completed the steps to set up Arrow AB test on your Shopify online store!