k-means clustering of armies

show games with more than a certain # of a certain unit at a certain time
filter by whether the match is from a frequent uploader

Author: dsjoerg

.gitignore

@@ -45,3 +45,5 @@ node_modules
 # Ignore temp data used for development
 public/ents*
 public/matches*
+public/minutes*
+public/frequent*

Gemfile

@@ -142,3 +142,5 @@ gem 'curb'
 #gem 'aws-s3'
 
 gem 'activemerchant'
+
+gem 'nokogiri', '1.6.3.1'

Gemfile.lock

@@ -160,7 +160,7 @@ GEM
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
     mime-types (1.25.1)
-    mini_portile (0.5.2)
+    mini_portile (0.6.0)
     money (5.1.1)
       i18n (~> 0.6.0)
     mono_logger (1.1.0)
@@ -170,8 +170,8 @@ GEM
     net-http-spy (0.2.1)
     netrc (0.7.7)
     newrelic_rpm (3.9.1.236)
-    nokogiri (1.6.1)
-      mini_portile (~> 0.5.0)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
     orm_adapter (0.4.0)
     paperclip (4.1.1)
       activemodel (>= 3.0.0)
@@ -309,6 +309,7 @@ DEPENDENCIES
   mysql2
   net-http-spy
   newrelic_rpm
+  nokogiri (= 1.6.3.1)
   paperclip
   paperclip-aws
   rails (= 3.2.19)

README.md

@@ -29,6 +29,7 @@ On Mac OSX, you can use homebrew as package manager: http://mxcl.github.com/home
 ### Basic installation and updating
 
  * Run Bundler (`bundle`)
+ ** On Mac OSX, you may run into hassle with nokogiri: http://stackoverflow.com/a/21616065
  * Copy and adjust database configuration (`cp config/database.yml.example config/database.yml`)
  * Copy secrets configuration (`cp config/secrets.yml.example config/secrets.yml`)
  * Copy s3 configuration (`cp config/s3.yml.example config/s3.yml`)

app/assets/javascripts/angular/controllers/scout.js

@@ -15,6 +15,74 @@ gg.controller('ScoutController', ['$scope', '$element', '$urlFilter',
       console.log('refresh!', params);
     }
 
+    NUM_CLUSTERS = 10;
+
+    $scope.compute_groups = function() {
+	var start1 = Date.now();
+	var start2 = Date.now();
+	var all_gamerecords = dateDim.top(Infinity);
+	var armies = _.map(all_gamerecords, function(gr) {
+		return gr.player.army_vector(8);
+	});
+	var start3 = Date.now();
+        kmeans_output = clusterfck.kmeans(armies, NUM_CLUSTERS);
+	var end = Date.now();
+	var total_time = end - start1;
+	console.log("kmeans took " + (total_time / 1000) + " seconds");
+
+	var centroids = kmeans_output[0];
+	var clusters = kmeans_output[1];
+	var assignment = kmeans_output[2];
+
+	// make groups an array of empty arrays, one for each cluster
+	var cluster_gamerecords = _.map(clusters, function(){return []});
+
+	// put the gamerecords for each cluster into its respective array
+	_.each(assignment, function(gr_assign, index) {
+		cluster_gamerecords[gr_assign].push(all_gamerecords[index]);
+	    });
+
+	// then compute $scope.groups as aggregate functions of arrays of gamerecords
+	var groupstats = _.map(clusters, function(cluster, cluster_index) {
+		var groupstat = { count: cluster_gamerecords[cluster_index].length,
+		                  gamerecords: cluster_gamerecords[cluster_index]
+		};
+		var armySum = _.map(_.range(0,19), function() { return 0.0 });
+		var numWins = 0;
+		_.each(cluster_gamerecords[cluster_index], function(gamerecord) {
+			_.each(_.range(0,19), function(army_index) {
+				armySum[army_index] += gamerecord.player.stat(8, 'u' + army_index);
+			    });
+			if (gamerecord.player.win == 'true') {
+			    numWins++;
+			}
+		    });
+		armySum = _.map(armySum, function(unit_count) {
+			var average_count = unit_count / groupstat.count;
+			if (average_count < 0.1) {
+			    return '';
+			}
+			if (average_count >= 10.0) {
+			    return Math.round(average_count);
+			}
+			return Math.floor(average_count * 10.0) / 10.0;
+
+		    });
+		groupstat.unit_counts = armySum;
+		groupstat.winPct = Math.floor(100.0 * numWins / groupstat.count);
+		groupstat.click = function() {
+		    //		    console.log("groupstat clicked!", this);
+		    $scope.groupsub = this.gamerecords;
+		};
+		
+		return groupstat;
+	    });
+
+	$scope.groupstats = _.sortBy(groupstats, function(groupstat) {
+		return -1 * groupstat.winPct;
+	    });
+    }
+
     $scope.$watch('race + vs_race', function(v) {
       $scope.filter.params.race = $scope.race;
       $scope.filter.params.vs_race = $scope.vs_race;

app/assets/javascripts/clusterfck/kmeans.js

@@ -0,0 +1,106 @@
+clusterfck = {}
+
+clusterfck.distances = {
+  euclidean: function(v1, v2) {
+      var total = 0;
+      for (var i = 0; i < v1.length; i++) {
+         total += Math.pow(v2[i] - v1[i], 2);      
+      }
+      return Math.sqrt(total);
+   },
+   manhattan: function(v1, v2) {
+     var total = 0;
+     for (var i = 0; i < v1.length ; i++) {
+        total += Math.abs(v2[i] - v1[i]);      
+     }
+     return total;
+   },
+   max: function(v1, v2) {
+     var max = 0;
+     for (var i = 0; i < v1.length; i++) {
+        max = Math.max(max , Math.abs(v2[i] - v1[i]));      
+     }
+     return max;
+   }
+};
+
+function randomCentroids(points, k) {
+   var centroids = points.slice(0); // copy
+   centroids.sort(function() {
+      return (Math.round(Math.random()) - 0.5);
+   });
+   return centroids.slice(0, k);
+}
+
+function closestCentroid(point, centroids, distance) {
+   var min = Infinity,
+       index = 0;
+   for (var i = 0; i < centroids.length; i++) {
+      var dist = distance(point, centroids[i]);
+      if (dist < min) {
+         min = dist;
+         index = i;
+      }
+   }
+   return index;
+}
+
+function kmeans(points, k, distance, snapshotPeriod, snapshotCb) {
+   k = k || Math.max(2, Math.ceil(Math.sqrt(points.length / 2)));
+
+   distance = distance || "euclidean";
+   if (typeof distance == "string") {
+      distance = clusterfck.distances[distance];
+   }
+
+   var centroids = randomCentroids(points, k);
+   var assignment = new Array(points.length);
+   var clusters = new Array(k);
+
+   var iterations = 0;
+   var movement = true;
+   while (movement) {
+      // update point-to-centroid assignments
+      for (var i = 0; i < points.length; i++) {
+         assignment[i] = closestCentroid(points[i], centroids, distance);
+      }
+
+      // update location of each centroid
+      movement = false;
+      for (var j = 0; j < k; j++) {
+         var assigned = [];
+         for (var i = 0; i < assignment.length; i++) {
+            if (assignment[i] == j) {
+               assigned.push(points[i]);
+            }
+         }
+
+         if (!assigned.length) {
+            continue;
+         }
+         var centroid = centroids[j];
+         var newCentroid = new Array(centroid.length);
+
+         for (var g = 0; g < centroid.length; g++) {
+            var sum = 0;
+            for (var i = 0; i < assigned.length; i++) {
+               sum += assigned[i][g];
+            }
+            newCentroid[g] = sum / assigned.length;
+
+            if (newCentroid[g] != centroid[g]) {
+               movement = true;
+            }
+         }
+         centroids[j] = newCentroid;
+         clusters[j] = assigned;
+      }
+
+      if (snapshotCb && (iterations++ % snapshotPeriod == 0)) {
+         snapshotCb(clusters);
+      }
+   }
+   return [centroids, clusters, assignment];
+}
+
+clusterfck.kmeans = kmeans;

app/assets/javascripts/gg.js

@@ -73,6 +73,7 @@
 //= require d3.js
 //= require crossfilter.js
 //= require scout.js
+//= require ./clusterfck/kmeans.js
 
 function dontDoThisEither() {
     //  Proxino.log("log this");