LockerProject · quartzjer · Oct 2, 2011 · Oct 2, 2011 · Oct 14, 2011 · Oct 21, 2011
diff --git a/Apps/HelloTimeline/img/facebook.png b/Apps/HelloTimeline/img/facebook.png
diff --git a/Apps/HelloTimeline/img/foursquare.png b/Apps/HelloTimeline/img/foursquare.png
diff --git a/Apps/HelloTimeline/img/instagram.png b/Apps/HelloTimeline/img/instagram.png
diff --git a/Apps/HelloTimeline/img/twitter.png b/Apps/HelloTimeline/img/twitter.png
diff --git a/Apps/HelloTimeline/index.html b/Apps/HelloTimeline/index.html
@@ -0,0 +1,34 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+
+  <title>Timeline Viewer</title>
+
+  <style type="text/css">
+body {
+  margin: 40px;
+}
+
+h1 {
+  font-size: 24pt;
+  text-align: center;
+  margin-bottom:40px;
+}
+  </style>
+
+</head>
+
+<body>
+  <span id='status'></span>
+  <h1>Forever Timeless</h1>
+  <ul id="test" class="items"></ul>
+  <button id="moar">Load more</button>
+
+</body>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script>
+<script src="script.js"></script>
+<script src='/static/js/viewer.js'></script> <!-- optional, only loaded when hosted, common locker viewer utilities -->
+</html>
diff --git a/Apps/HelloTimeline/package.json b/Apps/HelloTimeline/package.json
@@ -0,0 +1,31 @@
+{
+  "author": "Singly, Inc. <[email protected]>",
+  "name": "hellotimeline",
+  "description": "A simple way to view your timeline.",
+  "version": "0.0.1",
+  "homepage": "https://github.com/LockerProject/locker",
+  "repository": {
+    "type": "app",
+    "title": "Hello Timeline",
+    "status": "stable",
+    "static": true,
+    "hidden": true,
+    "handle": "hellotimeline",
+    "update": true,
+    "uses": {
+      "services": [
+        "facebook",
+        "foursquare",
+        "instagram",
+        "twitter"
+      ]
+    },
+    "github": "https://github.com/LockerProject/locker",
+    "url": ""
+  },
+  "engines": {
+    "node": ">=0.4.9"
+  },
+  "dependencies": {},
+  "devDependencies": {}
+}
diff --git a/Apps/HelloTimeline/script.js b/Apps/HelloTimeline/script.js
@@ -0,0 +1,103 @@
+var baseUrl = false;
+
+$(document).ready(function() {
+    if(baseUrl === false) window.alert("Couldn't find your locker, you might need to add a config.js (see dev.singly.com)");
+});
+
+var offset=0;
+$(function() {
+    // be careful with the limit, some people have large datasets ;)
+    loadStuff();
+    loadStatus();
+    $("#moar").click( function(){
+        offset += 50;
+        loadStuff();
+    });
+});
+
+var max = 20;
+function loadStatus()
+{
+    $.getJSON(baseUrl + '/Me/timeline/state',{}, function(data) {
+        if(!data) return $("#status").text("timeline failed :(");
+        if(data.ready == 1) return $("#status").text("");
+        var name = (data.current) ? data.current.type + " at "+data.current.offset : "";
+        $("#status").text("timeline is indexing "+name);
+        if(max-- <= 0) return $("#status").text("plz reload");
+        window.setTimeout(loadStatus, 10000); // poll for a bit
+    });
+}
+
+function loadStuff(){
+    $.getJSON(baseUrl + '/Me/timeline/',{limit:50, offset:offset}, function(data) {
+        if(!data || !data.length) return;
+        var html = '<h4>'+ago(data[0].first)+'</h4>';
+        for(var i in data)
+        {
+            var p = data[i];
+            p.refs.forEach(function(ref){ html += icon(ref); });
+            var resp = p.comments + p.ups;
+            html += "<i>"+p.from.name+"</i>: <span title='"+JSON.stringify(p,null,'\t')+"'>" + p.text + '</span>';
+            html += ' <a href="'+baseUrl+'/Me/timeline/ref?id='+escape(p.ref)+'">.js</a>';
+            if(resp > 0) html +=  "<div id='"+p.id+"'><a href='javascript:loadResp(\""+p.id+"\")'>["+resp+"]</a></div> ";
+            html += '<hr>';
+        }
+        $("#test").append(html);
+    });
+}
+
+function loadResp(id)
+{
+    $.getJSON(baseUrl + '/Me/timeline/getResponses',{item:id}, function(data) {
+        $("#"+id).html("");
+        if(!data || !data.length) return;
+        var ups = " ups: ";
+        var coms = "";
+        for(var i in data)
+        {
+            var r = data[i];
+            if(r.type == "up")
+            {
+                ups += icon(r.ref);
+                ups += " "+r.from.name+", ";
+            }
+            if(r.type == "comment")
+            {
+                coms += "<br>"
+                coms += icon(r.ref);
+                coms += " "+r.from.name+": ";
+                coms += r.text;
+            }
+        }
+        $("#"+id).append(ups+coms);
+    });
+}
+
+function icon(ref)
+{
+    var start = ref.indexOf('//')+2;
+    var net = ref.substr(start,ref.indexOf('/',start+2)-start);
+    if(net == "links") return "";
+    return "<img src='img/"+net+".png' width=16 height=16 />";
+
+}
+
+function ago(at)
+{
+    var tip = '';
+    var timeDiff = Date.now() - at;
+    if (timeDiff < 60000) {
+        tip += 'last updated less than a minute ago';
+    } else if (timeDiff < 3600000) {
+        tip += 'last updated ' + Math.floor(timeDiff / 60000) + ' minutes ago';
+    } else if (timeDiff < 43200000) {
+        tip += 'last updated over an hour ago';
+    } else if (timeDiff < 43800000) {
+        tip += 'last updated ' + Math.floor(timeDiff / 3600000) + ' hours ago';
+    } else {
+        var d = new Date;
+        d.setTime(at);
+        tip += 'last updated ' + d.toString();
+    }
+    return tip;
+}
diff --git a/Apps/prolific_posters b/Apps/prolific_posters
diff --git a/Collections/Links/api.js b/Collections/Links/api.js
@@ -156,6 +156,32 @@ module.exports = function(app, lockerInfo) {
     });
   });
 
+  // get just encounters raw!
+  // expose way to get raw links and encounters
+  app.get('/encounters', function(req, res) {
+      var options = {sort:{"_id":-1}};
+      if(!req.query["all"]) options.limit = 20; // default 20 unless all is set
+      if (req.query.limit) {
+          options.limit = parseInt(req.query.limit);
+      }
+      if (req.query.offset) {
+          options.offset = parseInt(req.query.offset);
+      }
+      if(req.query['stream'] == "true")
+      {
+          res.writeHead(200, {'content-type' : 'application/jsonstream'});
+      }
+      var results = [];
+      dataStore.getEncounters(options, function(item) {
+          if(req.query['stream'] == "true") return res.write(JSON.stringify(item)+'\n');
+          results.push(item);
+      }, function(err){
+          if(err) logger.error(err);
+          if(req.query['stream'] == "true") return res.end();
+          return res.send(results);
+      });
+  });
+
 }
 
 function isFull(full) {

diff --git a/Collections/Links/dataIn.js b/Collections/Links/dataIn.js
@@ -109,10 +109,11 @@ var encounterQueue = async.queue(function(e, callback) {
         async.forEach(urls,function(u,cb){
             linkMagic(u,function(link){
                 // make sure to pass in a new object, asyncutu
-                dataStore.addEncounter(lutil.extend(true,{orig:u,link:link},e), function(err,doc){
+                dataStore.addEncounter(lutil.extend(true,{orig:u,link:link.link},e), function(err,doc){
                     if(err) return cb(err);
                     dataStore.updateLinkAt(doc.link, doc.at, function(err, obj){
                         if(err) return cb(err);
+                        obj.encounters = [doc]; // include this encounter!
                         locker.ievent(lutil.idrNew("link","links",obj.id),obj,"update"); // let happen independently
                         cb();
                     });
@@ -140,6 +141,7 @@ function linkMagic(origUrl, callback){
     dataStore.checkUrl(origUrl,function(linkUrl){
         if(linkUrl) return callback(linkUrl); // short circuit!
         // new one, expand it to a full one
+        var linkUrl;
         util.expandUrl({url:origUrl},function(u2){linkUrl=u2},function(){
            // fallback use orig if errrrr
            if(!linkUrl) {
@@ -149,7 +151,7 @@ function linkMagic(origUrl, callback){
            // does this full one already have a link stored?
            dataStore.getLinks({link:linkUrl,limit:1},function(l){link=l},function(err){
               if(link) {
-                  return callback(link.link); // yeah short circuit dos!
+                  return callback(link); // yeah short circuit dos!
               }
               // new link!!!
               link = {link:linkUrl};
@@ -164,11 +166,13 @@ function linkMagic(origUrl, callback){
                           if (!link.at) link.at = Date.now();
                           dataStore.addLink(link,function(err, obj){
                               locker.ievent(lutil.idrNew("link","links",obj.id),obj); // let happen independently
-                              callback(link.link); // TODO: handle when it didn't get stored or is empty better, if even needed
+                              callback(link); // TODO: handle when it didn't get stored or is empty better, if even needed
                               // background fetch oembed and save it on the link if found
                               oembed.fetch({url:link.link, html:html}, function(e){
                                   if(!e) return;
-                                  dataStore.updateLinkEmbed(link.link, e, function(){});
+                                  dataStore.updateLinkEmbed(link.link, e, function(err, obj){
+                                      locker.ievent(lutil.idrNew("link","links",obj.id),obj,"update");
+                                  });
                               });
                           });
                       });
@@ -205,7 +209,7 @@ function getEncounterTwitter(event)
 {
     var tweet = event.data;
     var txt = (tweet.retweeted_status && tweet.retweeted_status.text) ? tweet.retweeted_status.text : tweet.text;
-    var e = {id:tweet.id
+    var e = {id:tweet.id_str
         , idr:event.idr
         , network:"twitter"
         , text: txt + " " + tweet.user.screen_name

diff --git a/Collections/Links/dataStore.js b/Collections/Links/dataStore.js
@@ -72,7 +72,7 @@ exports.checkUrl = function(origUrl, callback) {
         if(err) return callback();
         cursor.nextObject(function(err, item){
             if(err || !item || !item.link) return callback();
-            callback(item.link);
+            callback(item);
         });
     });
 }
@@ -93,7 +93,7 @@ exports.getLinks = function(arg, cbEach, cbDone) {
     linkCDS.findWrap(f,arg,cbEach,cbDone);
 }
 
-exports.getFullLink = function(id, cbDone) {
+exports.getLink = function(id, cbDone) {
     var link = null;
     exports.getLinks({link:id}, function(l) { link = l; }, function() { cbDone(link); });
 }

diff --git a/Collections/Timeline/README.md b/Collections/Timeline/README.md
@@ -0,0 +1,19 @@
+The "feeds" collection is all of the socially shared items across any service, facebook newsfeed, twitter timeline, foursquare recents, etc.
+
+It consists of a primary type called an "item" which can have an array of "responses" that are comments, likes, retweets, etc from other people.
+
+The goal is to dedup cross-posts to multiple networks from one person, particularly things like a foursquare checkin.
+
+
+Notes:
+
+  id references and keys
+  cross-collection reference storage with links
+  link based dedup of foursquare
+  dedup uses keys as guids across networks and for text matching variations
+  /update takes type arg for individual updates
+  idrs as via, no original storage
+  prioritization scheme in merging items
+  generic from object, list of froms
+  references array of all sources
+  generalized responses and top-level statistics