Setting up Arrow AB Test on Shopify store

For a more data-driven approach of adopting Arrow and seeing the impact of conversion uplift, merchants will have to set up AB testing of Arrow on their online stores. A very general set up is a 50:50 split, where 50% of the traffic sees the original set up (ie. current checkout buttons which direct customers to the current checkout experience) vs another 50% that sees the set up installed with Arrow (ie. checkout buttons embedded with Arrow which direct customers to the Arrow checkout experience).

1. Initialise Mixpanel and store mixpanel.get_distinct_id value as cookie

Mixpanel is an analytics tool that tracks events and provide ease in maintaining identity across events tracked, so that user's journey across the platform can be stringed together and analysed. We initialise Mixpanel on your store and obtain the distinct_id that Mixpanel generates for the user, so that this same ID can be passed on to Arrow Checkout and have the subsequent events associated back to the same user.

Paste the below code at the section:

<script type="text/javascript">
(function(f,b){if(!b.__SV){var e,g,i,h;window.mixpanel=b;b._i=[];b.init=function(e,f,c){function g(a,d){var b=d.split(".");2==b.length&&(a=a[b[0]],d=b[1]);a[d]=function(){a.push([d].concat(Array.prototype.slice.call(arguments,0)))}}var a=b;"undefined"!==typeof c?a=b[c]=[]:c="mixpanel";a.people=a.people||[];a.toString=function(a){var d="mixpanel";"mixpanel"!==c&&(d+="."+c);a||(d+=" (stub)");return d};a.people.toString=function(){return a.toString(1)+".people (stub)"};i="disable time_event track track_pageview track_links track_forms track_with_groups add_group set_group remove_group register register_once alias unregister identify name_tag set_config reset opt_in_tracking opt_out_tracking has_opted_in_tracking has_opted_out_tracking clear_opt_in_out_tracking start_batch_senders people.set people.set_once people.unset people.increment people.append people.union people.track_charge people.clear_charges people.delete_user people.remove".split(" ");
for(h=0;h<i.length;h++)g(a,i[h]);var j="set set_once union unset remove delete".split(" ");a.get_group=function(){function b(c){d[c]=function(){call2_args=arguments;call2=[c].concat(Array.prototype.slice.call(call2_args,0));a.push([e,call2])}}for(var d={},e=["get_group"].concat(Array.prototype.slice.call(arguments,0)),c=0;c<j.length;c++)b(j[c]);return d};b._i.push([e,f,c])};b.__SV=1.2;e=f.createElement("script");e.type="text/javascript";e.async=!0;e.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?
MIXPANEL_CUSTOM_LIB_URL:"file:"===f.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";g=f.getElementsByTagName("script")[0];g.parentNode.insertBefore(e,g)}})(document,window.mixpanel||[]);

// initiate mixpanel function and store distinct id as cookie
  mixpanel.init("9ba527872ab64cef0d90d69567730db1", {
    loaded: function(mixpanel) {
        distinct_id = mixpanel.get_distinct_id();
        setCookie('mx_distinct_id', distinct_id, 90);
    }
  });

</script>

2. Generate AB testing ID

It is important for each unique user to be assigned to an experiment variant, and for his/her identity to be persisted in that same variant. In other words, once a user has been assigned to 'Group A', he should always be seeing the 'Group A' experience, vice versa.

We will generate a unique ID for the user and store it on his browser cookie. This ID will be used to obtain an assigned variant, and be used to uniquely identify this user on a given browser (device). Going forward, so long as we could retrieve a stored ID on the cookie, we always fall back to using the generated value.

<script>
  function uuidv4() {
   return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
     (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
   );
  }
  
  function setCookie(cname, cvalue, exdays) {
    const d = new Date();
    d.setTime(d.getTime() + (exdays * 24 * 60 * 60 * 1000));
    let expires = "expires="+d.toUTCString();
    document.cookie = cname + "=" + cvalue + ";" + expires + ";path=/";
 }

 function getCookie(cname) {
    let name = cname + "=";
    let ca = document.cookie.split(';');
    for(let i = 0; i < ca.length; i++) {
     let c = ca[i];
     while (c.charAt(0) == ' ') {
       c = c.substring(1);
     }
     if (c.indexOf(name) == 0) {
       return c.substring(name.length, c.length);
     }
    }
     return "";
  }
  
  var ab_user_id = getCookie('ab_user_id');
  var mx_distinct_id = getCookie('mx_distinct_id');
  
  if(ab_user_id == null || ab_user_id == "" || ab_user_id == undefined){
    ab_user_id = uuidv4();
    setCookie("ab_user_id", ab_user_id, 90);
  }
</script>

3. Call the AB variant assignment endpoint for variant assignment

Once the Mixpanel and AB testing IDs have been prepared, we call the AB variant assignment endpoint to obtain a variant assigned for this user.

The endpoint:

We would have to parse the response coming back from the endpoint - should the response informs show_arrow being false, we have to run a custom function to hide the Arrow button/show the standard checkout button.

<script>
$.ajax({
    type: 'post',
    url: 'https://fly.witharrow.co/api/arrow/arrow_abtest',
    dataType: 'json',
    data: {
      client_uuid : ab_user_id,
      merchant_id : 244 ,
    },
    cache: false,
    success: function(returndata) {
      console.log("response");
      console.log(returndata);
      if(returndata['data']['show_arrow'] === false){
         $("#arrow-checkout-block-cart").hide();
         changeCartPageCheckoutButtonText();
      }

      mxAfterAjax(returndata, 'CartPageReach', window.location.href, mx_distinct_id)
    },
    error: function() { 
      console.error('Failed to process ajax !');
    }

  });
</script>

4. Fire the Mixpanel event bringing along with it the 'Variant ID'

Once the variant assignment has been obtained and the corresponding experience has been delivered, we would need to fire the event to Mixpanel for tracking purposes. The event that we fire is usually either CartPageReach or ProductPageReach, since generally we are interested to track the conversion difference starting from the customer landing on the product page, through to completing payment.

It is important to pass along the Variant ID, given that it will allow us to easily split the funnel view based on 'Group A' vs 'Group B' and make performance comparison on Mixpanel. Remember to also pass along the mx_distinct_id that should have been obtained from cookie.

Have the below function added to section, so that it can be called from either cart/product pages:

function mxAfterAjax(data, eventName, location, mx_distinct_id){
  console.log(data, mx_distinct_id)
  var interval = setInterval(function () {
    if (mx_distinct_id.length > 0) {
      mixpanel.track(eventName, {
        'MerchantID': 6,
        'Origin' : 'E-Store',
        'VariantId' : data.data.variant_id,
        'PageUrl': location
      }); 
      clearInterval(interval);
    }
  }, 300);
}

From product page, it can be used like this:

.
.
.    
success: function(returndata) {
      if(returndata['data']['show_arrow'] === false){
         $("#arrow-checkout-block").hide();
      }
//    mixpanel tracking after get variant_id
      mxAfterAjax(returndata, 'ProductPageReach', window.location.href, mx_distinct_id)
    },

With that, you have completed the steps in setting up Arrow AB test on your Shopify online store.