coalesce events, truncate long string values

Author: djih

amplitude.js

@@ -128,6 +128,7 @@ var log = function(s) {
 
 var IDENTIFY_EVENT = '$identify';
 var API_VERSION = 2;
+var MAX_STRING_LENGTH = 1024;
 var DEFAULT_OPTIONS = {
   apiEndpoint: 'api.amplitude.com',
   cookieExpiration: 365 * 10,
@@ -149,6 +150,7 @@ var DEFAULT_OPTIONS = {
 };
 var LocalStorageKeys = {
   LAST_EVENT_ID: 'amplitude_lastEventId',
+  LAST_IDENTIFY_ID: 'amplitude_lastIdentifyId',
   LAST_EVENT_TIME: 'amplitude_lastEventTime',
   SESSION_ID: 'amplitude_sessionId'
 };
@@ -164,11 +166,14 @@ var Amplitude = function() {
 };
 
 Amplitude.prototype._eventId = 0;
+Amplitude.prototype._identifyId = 0;
 Amplitude.prototype._sending = false;
 Amplitude.prototype._lastEventTime = null;
 Amplitude.prototype._sessionId = null;
 Amplitude.prototype._newSession = false;
 
+Amplitude.prototype.Identify = Identify;
+
 /**
  * Initializes Amplitude.
  * apiKey The API Key for your app
@@ -225,10 +230,16 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
 
     if (this.options.saveEvents) {
       var savedUnsentEventsString = localStorage.getItem(this.options.unsentKey);
-      var savedUnsentIdentifysString = localStorage.getItem(this.options.unsentIdentifyKey);
       if (savedUnsentEventsString) {
         try {
           this._unsentEvents = JSON.parse(savedUnsentEventsString);
+        } catch (e) {
+          //log(e);
+        }
+      }
+      var savedUnsentIdentifysString = localStorage.getItem(this.options.unsentIdentifyKey);
+      if (savedUnsentIdentifysString) {
+        try {
           this._unsentIdentifys = JSON.parse(savedUnsentIdentifysString);
         } catch (e) {
           //log(e);
@@ -245,6 +256,7 @@ Amplitude.prototype.init = function(apiKey, opt_userId, opt_config) {
     this._lastEventTime = parseInt(localStorage.getItem(LocalStorageKeys.LAST_EVENT_TIME)) || null;
     this._sessionId = parseInt(localStorage.getItem(LocalStorageKeys.SESSION_ID)) || null;
     this._eventId = localStorage.getItem(LocalStorageKeys.LAST_EVENT_ID) || 0;
+    this._identifyId = localStorage.getItem(LocalStorageKeys.LAST_IDENTIFY_ID) || 0;
     var now = new Date().getTime();
     if (!this._sessionId || !this._lastEventTime || now - this._lastEventTime > this.options.sessionTimeout) {
       this._newSession = true;
@@ -267,6 +279,11 @@ Amplitude.prototype.nextEventId = function() {
   return this._eventId;
 };
 
+Amplitude.prototype.nextIdentifyId = function() {
+  this._identifyId++;
+  return this._identifyId;
+};
+
 // returns the number of unsent events and identifys
 Amplitude.prototype._unsentCount = function() {
   return this._unsentEvents.length + this._unsentIdentifys.length;
@@ -444,6 +461,34 @@ Amplitude.prototype.setVersionName = function(versionName) {
   }
 };
 
+// truncate string values in event and user properties so that request size does not get too large
+Amplitude.prototype._truncate = function(value) {
+  if (typeof(value) === 'object') {
+    if (Array.isArray(value)) {
+      for (var i = 0; i < value.length; i++) {
+        value[i] = this._truncate(value[i]);
+      }
+    } else {
+      for (var key in value) {
+        if (value.hasOwnProperty(key)) {
+          value[key] = this._truncate(value[key]);
+        }
+      }
+    }
+
+    return value;
+  }
+
+  return _truncateValue(value);
+};
+
+var _truncateValue = function(value) {
+  if (typeof(value) === 'string') {
+    return value.length > MAX_STRING_LENGTH ? value.substring(0, MAX_STRING_LENGTH) : value;
+  }
+  return value;
+};
+
 /**
  * Private logEvent method. Keeps apiProperties from being publicly exposed.
  */
@@ -459,16 +504,22 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
     return;
   }
   try {
+    var eventId;
+    if (eventType === IDENTIFY_EVENT) {
+      eventId = this.nextIdentifyId();
+      localStorage.setItem(LocalStorageKeys.LAST_IDENTIFY_ID, eventId);
+    } else {
+      eventId = this.nextEventId();
+      localStorage.setItem(LocalStorageKeys.LAST_EVENT_ID, eventId);
+    }
     var eventTime = new Date().getTime();
-    var eventId = this.nextEventId();
     var ua = this._ua;
     if (!this._sessionId || !this._lastEventTime || eventTime - this._lastEventTime > this.options.sessionTimeout) {
       this._sessionId = eventTime;
       localStorage.setItem(LocalStorageKeys.SESSION_ID, this._sessionId);
     }
     this._lastEventTime = eventTime;
     localStorage.setItem(LocalStorageKeys.LAST_EVENT_TIME, this._lastEventTime);
-    localStorage.setItem(LocalStorageKeys.LAST_EVENT_ID, eventId);
 
     // Add the utm properties, if any, onto the user properties.
     userProperties = userProperties || {};
@@ -498,8 +549,8 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
       device_model: ua.os.name || null,
       language: this.options.language,
       api_properties: apiProperties,
-      event_properties: eventProperties,
-      user_properties: userProperties,
+      event_properties: this._truncate(eventProperties),
+      user_properties: this._truncate(userProperties),
       uuid: UUID(),
       library: {
         name: 'amplitude-js',
@@ -508,14 +559,19 @@ Amplitude.prototype._logEvent = function(eventType, eventProperties, apiProperti
       // country: null
     };
 
-    //log('logged eventType=' + eventType + ', properties=' + JSON.stringify(eventProperties));
-
-    this._unsentEvents.push(event);
+    if (eventType === IDENTIFY_EVENT) {
+      this._unsentIdentifys.push(event);
+      if (this._unsentIdentifys.length > this.options.savedMaxCount) {
+        this._unsentIdentifys.splice(0, this._unsentIdentifys.length - this.options.savedMaxCount);
+      }
+    } else {
+      this._unsentEvents.push(event);
 
-    // Remove old events from the beginning of the array if too many
-    // have accumulated. Don't want to kill memory. Default is 1000 events.
-    if (this._unsentEvents.length > this.options.savedMaxCount) {
-      this._unsentEvents.splice(0, this._unsentEvents.length - this.options.savedMaxCount);
+      // Remove old events from the beginning of the array if too many
+      // have accumulated. Don't want to kill memory. Default is 1000 events.
+      if (this._unsentEvents.length > this.options.savedMaxCount) {
+        this._unsentEvents.splice(0, this._unsentEvents.length - this.options.savedMaxCount);
+      }
     }
 
     if (this.options.saveEvents) {
@@ -560,14 +616,26 @@ Amplitude.prototype.logRevenue = function(price, quantity, product) {
  * Remove events in storage with event ids up to and including maxEventId. Does
  * a true filter in case events get out of order or old events are removed.
  */
-Amplitude.prototype.removeEvents = function (maxEventId) {
-  var filteredEvents = [];
-  for (var i = 0; i < this._unsentEvents.length; i++) {
-    if (this._unsentEvents[i].event_id > maxEventId) {
-      filteredEvents.push(this._unsentEvents[i]);
+Amplitude.prototype.removeEvents = function (maxEventId, maxIdentifyId) {
+  if (maxEventId) {
+    var filteredEvents = [];
+    for (var i = 0; i < this._unsentEvents.length; i++) {
+      if (this._unsentEvents[i].event_id > maxEventId) {
+        filteredEvents.push(this._unsentEvents[i]);
+      }
+    }
+    this._unsentEvents = filteredEvents;
+  }
+
+  if (maxIdentifyId) {
+    var filteredIdentifys = [];
+    for (var j = 0; j < this._unsentIdentifys.length; j++) {
+      if (this._unsentIdentifys[j].event_id > maxIdentifyId) {
+        filteredIdentifys.push(this._unsentIdentifys[j]);
+      }
     }
+    this._unsentIdentifys = filteredIdentifys;
   }
-  this._unsentEvents = filteredEvents;
 };
 
 Amplitude.prototype.sendEvents = function(callback) {
@@ -577,10 +645,42 @@ Amplitude.prototype.sendEvents = function(callback) {
         this.options.apiEndpoint + '/';
 
     // Determine how many events to send and track the maximum event id sent in this batch.
-    var numEvents = Math.min(this._unsentEvents.length, this.options.uploadBatchSize);
-    var maxEventId = this._unsentEvents[numEvents - 1].event_id;
+    var numEvents = Math.min(this._unsentCount(), this.options.uploadBatchSize);
+
+    // coalesce events from both queues
+    var eventsToSend = [];
+    var eventIndex = 0;
+    var identifyIndex = 0;
+
+    while (eventsToSend.length < numEvents) {
+      var event;
+
+      // case 1: no identifys - grab from events
+      if (identifyIndex >= this._unsentIdentifys.length) {
+        event = this._unsentEvents[eventIndex++];
+
+      // case 2: no events - grab from identifys
+      } else if (eventIndex >= this._unsentEvents.length) {
+        event = this._unsentIdentifys[identifyIndex++];
+
+      // case 3: need to compare timestamps
+      } else {
+        if (this._unsentIdentifys[identifyIndex].timestamp <= this._unsentEvents[eventIndex].timestamp) {
+          event = this._unsentIdentifys[identifyIndex++];
+        } else {
+          event = this._unsentEvents[eventIndex++];
+        }
+      }
+
+      eventsToSend.push(event);
+    }
+
+    var maxEventId = eventIndex > 0 && this._unsentEvents.length > 0 ?
+      this._unsentEvents[eventIndex - 1].event_id : null;
+    var maxIdentifyId = identifyIndex > 0 && this._unsentIdentifys.length > 0 ?
+      this._unsentIdentifys[identifyIndex - 1].event_id : null;
+    var events = JSON.stringify(eventsToSend);
 
-    var events = JSON.stringify(this._unsentEvents.slice(0, numEvents));
     var uploadTime = new Date().getTime();
     var data = {
       client: this.options.apiKey,
@@ -596,7 +696,7 @@ Amplitude.prototype.sendEvents = function(callback) {
       try {
         if (status === 200 && response === 'success') {
           //log('sucessful upload');
-          scope.removeEvents(maxEventId);
+          scope.removeEvents(maxEventId, maxIdentifyId);
 
           // Update the event cache after the removal of sent events.
           if (scope.options.saveEvents) {
@@ -612,7 +712,8 @@ Amplitude.prototype.sendEvents = function(callback) {
           //log('request too large');
           // Can't even get this one massive event through. Drop it.
           if (scope.options.uploadBatchSize === 1) {
-            scope.removeEvents(maxEventId);
+            // if massive event is identify, still need to drop it
+            scope.removeEvents(maxEventId, maxIdentifyId);
           }
 
           // The server complained about the length of the request.