Moved most of git based dependencies to npm

README.md

@@ -104,7 +104,6 @@ Understanding Walnut
 │   ├── rest
 │   └── services
 └── var
-    ├── <<pkgname>>/config.json
     └── sites
 ```
 
@@ -280,7 +279,7 @@ The packages:
 The permissions:
 
 ```
-/srv/walnut/etc/
+/srv/walnut/packages/
 └── client-api-grants
     └── cloud.foobar.me
           '''

dist/var/example.com/config.json

@@ -1,11 +0,0 @@
-{ "mailgun.org": {
-    "apiKey": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-  , "apiPublicKey": "pubkey-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-  , "auth": {
-      "user": "mailer@example.com"
-    , "pass": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    , "api_key": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    , "domain": "example.com"
-    }
-  }
-}

installer/install.sh

@@ -9,8 +9,6 @@ my_name=walnut
 my_app_pkg_name=com.daplie.walnut.web
 my_app_ver="v1.2"
 my_azp_oauth3_ver="v1.2"
-# is the old version still needed in launchpad?
-#my_azp_oauth3_ver="v1.1.3"
 export NODE_VERSION="v8.9.0"
 
 if [ -z "${my_tmp-}" ]; then
@@ -26,8 +24,9 @@ fi
 ### IMPORTANT ###
 ###  VERSION  ###
 #my_app_ver="v1.1"
-my_app_ver="v1.2"
+my_app_ver="installer-v2"
 my_launchpad_ver="v1.2"
+my_azp_oauth3_ver="v1.1.3"
 my_iss_oauth3_rest_ver="v1.2.0"
 my_iss_oauth3_pages_ver="v1.2.1"
 my_www_daplie_ver=v1.0.15
@@ -107,20 +106,20 @@ pushd $my_tmp/opt/$my_name/core/lib/walnut@daplie.com/setup
   git pull
   git checkout $my_launchpad_ver
 
-  git clone https://git.oauth3.org/OAuth3/oauth3.js.git ./assets/oauth3.org
+  git clone https://git.daplie.com/OAuth3/oauth3.js.git ./assets/oauth3.org
   pushd assets/oauth3.org
     git checkout $my_azp_oauth3_ver
   popd
 popd
 
 pushd $my_tmp/opt/$my_name/packages
-  git clone https://git.oauth3.org/OAuth3/issuer.rest.walnut.js.git rest/issuer@oauth3.org
+  git clone https://git.daplie.com/OAuth3/issuer_oauth3.org.git rest/issuer@oauth3.org
   pushd rest/issuer@oauth3.org/
       git checkout $my_iss_oauth3_rest_ver
       $my_npm install
   popd
 
-  git clone https://git.oauth3.org/OAuth3/issuer.html.git pages/issuer@oauth3.org
+  git clone https://git.daplie.com/OAuth3/org.oauth3.git pages/issuer@oauth3.org
   pushd pages/issuer@oauth3.org
     git checkout $my_iss_oauth3_pages_ver
     bash ./install.sh

lib/apis.js

@@ -124,7 +124,6 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
       var tok = req.oauth3.token;
       var accountId = req.params.accountId || '__NO_ID_GIVEN__';
       var ppid;
-      var iss = tok.iss;
 
       if (tok.sub && tok.sub.split(/,/g).filter(function (ppid) {
         return ppid === accountId;
@@ -132,7 +131,6 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
         ppid = accountId;
       }
 
-      // Deprecated backwards compat. To be removed.
       if (tok.axs && tok.axs.filter(function (acc) {
         return acc.id === accountId || acc.appScopedId === accountId;
       }).length) {
@@ -147,9 +145,10 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
         return PromiseA.reject(new Error("missing accountId '" + accountId + "' in access token"));
       }
 
-      return req.oauth3.rescope().then(function (accountIdx) {
+      return req.oauth3.rescope(ppid).then(function (accountIdx) {
         req.oauth3.accountIdx = accountIdx;
         req.oauth3.ppid = ppid;
+        req.oauth3.accountHash = crypto.createHash('sha1').update(accountIdx).digest('hex');
         //console.log('[walnut@daplie.com] accountIdx:', accountIdx);
         //console.log('[walnut@daplie.com] ppid:', ppid);
 
@@ -161,31 +160,19 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
   }
 
   function accountRequired(req, res, next) {
-    console.log('[accountRequired] [enter]');
-
-    var myIss = req.experienceId;
-    var isPpid;
-
     // if this already has auth, great
     if (req.oauth3.ppid && req.oauth3.accountIdx) {
-      // except that if it's a ppid, we have to internally exchange it for the real token
-      isPpid = (myIss === req.oauth3.iss && myIss !== req.oauth3.azp);
-      if (!isPpid) {
-        console.log('[accountRequired] has token already');
-        console.log(req.oauth3);
-        console.log('');
       next();
       return;
     }
+
+    // being public does not disallow authentication
+    if (req.isPublic && !req.oauth3.encodedToken) {
+      next();
+      return;
     }
 
     if (!req.oauth3.encodedToken) {
-      // being public does not disallow authentication
-      if (req.isPublic) {
-        next();
-        return;
-      }
-
       rejectableRequest(
         req
       , res
@@ -200,7 +187,6 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
       var tok = req.oauth3.token;
       var ppid;
       var err;
-      var iss = tok.iss;
 
       if (tok.sub) {
         if (tok.sub.split(/,/g).length > 1) {
@@ -209,30 +195,25 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
         }
         ppid = tok.sub;
       }
+      else if (tok.axs && tok.axs.length) {
+        if (tok.axs.length > 1) {
+          err = new Error("more than one 'axs' specified in token (also, update to using 'sub' instead)");
+          return PromiseA.reject(err);
+        }
+        ppid = tok.axs[0].appScopedId || tok.axs[0].id;
+      }
+      else if (tok.acx) {
+        ppid = tok.acx.appScopedId || tok.acx.id || tok.acx;
+      }
 
       if (!ppid) {
         return PromiseA.reject(new Error("could not determine accountId from access token"));
       }
 
-      return req.oauth3.rescope().then(function (accountIdx) {
+      return req.oauth3.rescope(ppid).then(function (accountIdx) {
-        console.log('[accountRequired] req.oauth3');
-        console.log(accountIdx);
-
-        var sub = accountIdx.split('@')[0];
-        var iss = accountIdx.split('@')[1];
-        var id = sub + '@' + iss;
-
-        req.oauth3.profile = {
-          id: id
-        , sub: sub
-        , iss: iss
-        };
-        req.oauth3.id = id;
-        req.oauth3.sub = sub;
-        req.oauth3.iss = iss;
-
         req.oauth3.accountIdx = accountIdx;
         req.oauth3.ppid = ppid;
+        req.oauth3.accountHash = crypto.createHash('sha1').update(accountIdx).digest('hex');
 
         next();
       });

lib/oauth3.js

@@ -2,17 +2,15 @@
 
 var PromiseA = require('bluebird');
 
-function generateRescope(req, Models, decoded) {
+function generateRescope(req, Models, decoded, fullPpid, ppid) {
-  var fullPpid = decoded.sub+'@'+decoded.iss;
-  var ppid = decoded.sub;
   return function (/*sub*/) {
     // TODO: this function is supposed to convert PPIDs of different parties to some account
     // ID that allows application to keep track of permisions and what-not.
     console.log('[rescope] Attempting ', fullPpid);
     return Models.IssuerOauth3OrgGrants.find({ azpSub: fullPpid }).then(function (results) {
       if (results[0]) {
-        console.log('[rescope] lucky duck: got it on the 1st try');
+        console.log('[rescope] lukcy duck: got it on the 1st try');
-        return results;
+        return PromiseA.resolve(results);
       }
 
       // XXX BUG XXX
@@ -21,46 +19,71 @@ function generateRescope(req, Models, decoded) {
     }).then(function (results) {
       var result = results[0];
 
-      if (!result || !result.sub || !decoded.iss) {
-        console.log('[rescope] Not a 2nd party token...');
-        return Models.IssuerOauth3OrgProfiles.get(fullPpid);
-      }
-
-      return result;
-    }).then(function (result) {
-      var err;
-
       if (!result || !result.sub || !decoded.iss) {
         // XXX BUG XXX TODO swap this external ppid for an internal (and ask user to link with existing profile)
         //req.oauth3.accountIdx = fullPpid;
-        console.log('[DEBUG] decoded:');
+        throw new Error("internal / external ID swapping not yet implemented. TODO: "
-        console.log(decoded);
+          + "No profile found with that credential. Would you like to create a new profile or link to an existing profile?");
-        console.log('[DEBUG] decoded.iss:', decoded.iss);
-        console.log('[DEBUG] fullPpid:', fullPpid);
-        console.log('[DEBUG] ppid:', ppid);
-
-        err = new Error(
-          "TODO: No profile found with that credential. Would you like to create a new profile or link to an existing profile?"
-        );
-        err.code = "E_NO_PROFILE@oauth3.org"
-        throw err;
-        //return req.oauth3.token.sub + '@' + req.oauth3.token.iss;
       }
 
       // XXX BUG XXX need to pass own url in to use as issuer for own tokens
-      req.oauth3.accountIdx = result.sub + '@' + (result.iss || decoded.iss);
+      req.oauth3.accountIdx = result.sub + '@' + decoded.iss;
 
       console.log('[rescope] result:');
-      console.log(result);
+      console.log(results);
-      console.log('[rescope] req.oauth3.accountIdx:', req.oauth3.accountIdx);
+      console.log(req.oauth3.accountIdx);
 
-      return req.oauth3.accountIdx;
+      return PromiseA.resolve(req.oauth3.accountIdx);
     });
   };
 }
 
-function verifyToken(token, opts) {
+function extractAccessToken(req) {
-  opts = opts || { audiences: [], complete: false };
+  var token = null;
+  var parts;
+  var scheme;
+  var credentials;
+
+  if (req.headers && req.headers.authorization) {
+    // Works for all of Authorization: Bearer {{ token }}, Token {{ token }}, JWT {{ token }}
+    parts = req.headers.authorization.split(' ');
+
+    if (parts.length !== 2) {
+      return PromiseA.reject(new Error("malformed Authorization header"));
+    }
+
+    scheme = parts[0];
+    credentials = parts[1];
+
+    if (-1 !== ['token', 'bearer'].indexOf(scheme.toLowerCase())) {
+      token = credentials;
+    }
+  }
+
+  if (req.body && req.body.access_token) {
+    if (token) { PromiseA.reject(new Error("token exists in header and body")); }
+    token = req.body.access_token;
+  }
+
+  // TODO disallow query with req.method === 'GET'
+  // NOTE: the case of DDNS on routers requires a GET and access_token
+  // (cookies should be used for protected static assets)
+  if (req.query && req.query.access_token) {
+    if (token) { PromiseA.reject(new Error("token already exists in either header or body and also in query")); }
+    token = req.query.access_token;
+  }
+
+  /*
+  err = new Error(challenge());
+  err.code = 'E_BEARER_REALM';
+
+  if (!token) { return PromiseA.reject(err); }
+  */
+
+  return PromiseA.resolve(token);
+}
+
+function verifyToken(token) {
   var jwt = require('jsonwebtoken');
   var decoded;
 
@@ -75,7 +98,6 @@ function verifyToken(token, opts) {
   try {
     decoded = jwt.decode(token, {complete: true});
   } catch (e) {}
-
   if (!decoded) {
     return PromiseA.reject({
       message: 'provided token not a JSON Web Token'
@@ -108,27 +130,6 @@ function verifyToken(token, opts) {
     });
   }
 
-  var audMatch = decoded.payload.aud && ('*' === decoded.payload.aud || opts.audiences.some(function (aud) { return -1 !== decoded.payload.aud.split(',').indexOf(aud); }));
-  var azpMatch = decoded.payload.azp && ('*' === decoded.payload.azp || opts.audiences.some(function (aud) { return -1 !== decoded.payload.azp.split(',').indexOf(aud); }));
-
-  if (!audMatch) {
-    console.log("[verifyToken] 'aud' '" + decoded.payload.aud + "' does not match '" + opts.audiences.join(',') + "'");
-  }
-  // TODO needs an option to verify that the sender of the token was, in fact, the azp (i.e. the Origin and/or Referer Headers)
-  if (!azpMatch) {
-    console.log("[verifyToken] 'azp' '" + decoded.payload.azp + "' does not match '" + opts.audiences.join(',') + "'");
-  }
-
-  if (!audMatch && !azpMatch) {
-    err = new Error(
-      "Application '" + req.experienceId + "' refused token because '" + decoded.payload.aud + "' is not an accepted audience (aud)"
-    + " and '" + decoded.payload.azp + "' is not an authorized party (azp)"
-    );
-    err.code = 'E_TOKEN_AUD';
-    err.url = 'https://oauth3.org/docs/errors#E_TOKEN_AUD'
-    return PromiseA.reject(err);
-  }
-
   var OAUTH3 = require('oauth3.js');
   OAUTH3._hooks = require('oauth3.js/oauth3.node.storage.js');
   return OAUTH3.discover(decoded.payload.iss).then(function (directives) {
@@ -187,27 +188,16 @@ function verifyToken(token, opts) {
     if (res.data.error) {
       return PromiseA.reject(res.data.error);
     }
-    var opts2 = {};
+    var opts = {};
     if (Array.isArray(res.data.alg)) {
-      opts2.algorithms = res.data.alg;
+      opts.algorithms = res.data.alg;
     } else if (typeof res.data.alg === 'string') {
-      opts2.algorithms = [res.data.alg];
+      opts.algorithms = [res.data.alg];
     }
 
     try {
-      if (opts.complete) {
+      return jwt.verify(token, require('jwk-to-pem')(res.data), opts);
-        opts2.complete = true;
-      }
-      return jwt.verify(token, require('jwk-to-pem')(res.data), opts2);
     } catch (err) {
-      if ('TokenExpiredError' === err.name) {
-        return PromiseA.reject({
-          message: 'TokenExpiredError: jwt expired'
-        , code: 'E_TOKEN_EXPIRED'
-        , url: 'https://oauth3.org/docs/errors#E_TOKEN_EXPIRED'
-        });
-      }
-
       return PromiseA.reject({
         message: 'token verification failed'
       , code: 'E_INVALID_TOKEN'
@@ -227,47 +217,32 @@ function deepFreeze(obj) {
   Object.freeze(obj);
 }
 
-function fiddleOauth3(Models, req) {
+function cookieOauth3(Models, req, res, next) {
-  var token = req.oauth3.encodedToken;
+  req.oauth3 = {};
 
-  req.oauth3.verifyAsync = function (jwt, opts) {
+  var token = req.cookies.jwt;
-    return verifyToken(jwt || token, opts || { audiences: [ req.experienceId ] });
+
+  req.oauth3.encodedToken = token;
+  req.oauth3.verifyAsync = function (jwt) {
+    return verifyToken(jwt || token);
   };
 
-  if (!token) {
+  return verifyToken(token).then(function  (decoded) {
-    return PromiseA.resolve(null);
-  }
-
-  return verifyToken(token, { complete: false, audiences: [ req.experienceId ] }).then(function  (decoded) {
-    var err;
     req.oauth3.token = decoded;
-
     if (!decoded) {
       return null;
     }
 
-    req.oauth3.ppid = decoded.sub;
+    var ppid = decoded.sub || decoded.ppid || decoded.appScopedId;
+    req.oauth3.ppid = ppid;
+    req.oauth3.accountIdx = ppid+'@'+decoded.iss;
 
-    req.oauth3.id = decoded.sub + '@' + decoded.iss;
+    var hash = require('crypto').createHash('sha256').update(req.oauth3.accountIdx).digest('base64');
-    req.oauth3.sub = decoded.sub;
+    hash = hash.replace(/\+/g, '-').replace(/\//g, '_').replace(/\=+/g, '');
-    req.oauth3.iss = decoded.iss;
+    req.oauth3.accountHash = hash;
-    req.oauth3.azp = decoded.azp;
-    req.oauth3.aud = decoded.aud;
 
-    req.oauth3.accountIdx = req.oauth3.id;
+    req.oauth3.rescope = generateRescope(req, Models, decoded, fullPpid, ppid);
-
+  }).then(function () {
-    req.oauth3.rescope = generateRescope(req, Models, decoded);
-  });
-}
-
-function cookieOauth3(Models, req, res, next) {
-  req.oauth3 = {};
-
-  var cookieName = 'jwt';
-  var token = req.cookies[cookieName];
-
-  req.oauth3.encodedToken = token;
-  fiddleOauth3(Models, req).then(function () {
     deepFreeze(req.oauth3);
     //Object.defineProperty(req, 'oauth3', {configurable: false, writable: false});
     next();
@@ -276,11 +251,6 @@ function cookieOauth3(Models, req, res, next) {
       next();
       return;
     }
-    if ('E_TOKEN_EXPIRED' === err.code) {
-      res.clearCookie(cookieName);
-      next();
-      return;
-    }
     console.error('[walnut] cookie lib/oauth3 error:');
     console.error(err);
     res.send(err);
@@ -290,49 +260,37 @@ function cookieOauth3(Models, req, res, next) {
 function attachOauth3(Models, req, res, next) {
   req.oauth3 = {};
 
-  var token = null;
+  extractAccessToken(req).then(function (token) {
-  var parts;
-  var scheme;
-  var credentials;
-
-  if (req.headers && req.headers.authorization) {
-    // Works for all of Authorization: Bearer {{ token }}, Token {{ token }}, JWT {{ token }}
-    parts = req.headers.authorization.split(' ');
-
-    if (parts.length !== 2) {
-      return PromiseA.reject(new Error("malformed Authorization header"));
-    }
-
-    scheme = parts[0];
-    credentials = parts[1];
-
-    if (-1 !== ['token', 'bearer'].indexOf(scheme.toLowerCase())) {
-      token = credentials;
-    }
-  }
-
-  if (req.body && req.body.access_token) {
-    if (token) { PromiseA.reject(new Error("token exists in header and body")); }
-    token = req.body.access_token;
-  }
-
-  // TODO disallow query with req.method === 'GET'
-  // NOTE: the case of DDNS on routers requires a GET and access_token
-  // (cookies should be used for protected static assets)
-  if (req.query && req.query.access_token) {
-    if (token) { PromiseA.reject(new Error("token already exists in either header or body and also in query")); }
-    token = req.query.access_token;
-  }
-
-  /*
-  err = new Error(challenge());
-  err.code = 'E_BEARER_REALM';
-
-  if (!token) { return PromiseA.reject(err); }
-  */
-
     req.oauth3.encodedToken = token;
-  fiddleOauth3(Models, req).then(function () {
+    req.oauth3.verifyAsync = function (jwt) {
+      return verifyToken(jwt || token);
+    };
+
+    if (!token) {
+      return null;
+    }
+    return verifyToken(token);
+  }).then(function  (decoded) {
+    req.oauth3.token = decoded;
+    if (!decoded) {
+      return null;
+    }
+
+    var ppid = decoded.sub || decoded.ppid || decoded.appScopedId;
+    var fullPpid = ppid+'@'+decoded.iss;
+    req.oauth3.ppid = ppid;
+
+    // TODO we can anonymize the relationship between our user as the other service's user
+    // in our own database by hashing the remote service's ppid and using that as the lookup
+    var hash = require('crypto').createHash('sha256').update(fullPpid).digest('base64');
+    hash = hash.replace(/\+/g, '-').replace(/\//g, '_').replace(/\=+/g, '');
+    req.oauth3.accountHash = hash;
+
+    req.oauth3.rescope = generateRescope(req, Models, decoded, fullPpid, ppid);
+
+    console.log('############### assigned req.oauth3:');
+    console.log(req.oauth3);
+  }).then(function () {
     //deepFreeze(req.oauth3);
     //Object.defineProperty(req, 'oauth3', {configurable: false, writable: false});
     next();

package.json

@@ -42,7 +42,7 @@
   "dependencies": {
     "bluebird": "3.x",
     "body-parser": "1.x",
-    "cluster-store": "git+https://git.daplie.com/Daplie/cluster-store.git#v2",
+    "cluster-store": "2.0.x",
     "connect": "3.x",
     "connect-cors": "0.5.x",
     "connect-recase": "^1.0.2",
@@ -57,17 +57,17 @@
     "jwk-to-pem": "^1.2.6",
     "mailchimp-api-v3": "^1.7.0",
     "mandrill-api": "^1.0.45",
-    "masterquest-sqlite3": "git+https://git.daplie.com/node/masterquest-sqlite3.git",
+    "masterquest-sqlite3": "1.0.x",
     "mkdirp": "^0.5.1",
     "multiparty": "^4.1.3",
     "nodemailer": "^1.4.0",
     "nodemailer-mailgun-transport": "1.x",
-    "oauth3.js": "git+https://git.oauth3.org/OAuth3/oauth3.js.git#v1.2",
+    "oauth3.js": "git+https://git.oauth3.org/OAuth3/oauth3.js.git",
     "recase": "^1.0.4",
     "request": "^2.81.0",
     "scmp": "^2.0.0",
     "serve-static": "1.x",
-    "sqlite3-cluster": "git+https://git.daplie.com/coolaj86/sqlite3-cluster.git#v2",
+    "sqlite3-cluster": "2.1.x",
     "stripe": "^4.22.0",
     "twilio": "1.x"
   }