merge

use updated oauth3

See merge request !4

.gitignore

@@ -1,4 +1,5 @@
 .*.sw*
+.dat
 
 # Logs
 logs

API.md

@@ -0,0 +1,244 @@
+* Bootstrap Initialization
+* Package Discovery
+* Package Layout
+* Package APIs
+* RESTful API constraints
+
+Bootstrap Initialization
+--------------
+
+Before walnut is configured it starts up in a bootstrap mode with a single API exposed to set its primary domain.
+
+```bash
+# Set up with example.com as the primary domain
+curl -X POST http://api.localhost.daplie.me:3000/api/walnut@daplie.com/init \
+  -H 'X-Forwarded-Proto: https' \
+  -H 'Content-Type: application/json' \
+  -d '{ "domain": "example.com" }'
+```
+
+From this point forward you can now interact with Walnut at that domain.
+
+OAuth3 Package Discovery
+-----------------
+
+Unlike most package systems such as npm (node.js), gem (ruby), pip (python), etc,
+which rely on a single, [centralized closed-source repository](https://github.com/npm/registry/issues/41),
+walnut packages use the OAuth3 Package Specification which allows for open and closed,
+public and private, free and paid packages, according to the desire of the publisher.
+
+In this model the name of a package is all that is necessary to install it from its publisher.
+
+Let's `hello@example.com` as an example:
+
+`hello@example.com` specifies that `hello` is a submodule of the `example.com` package.
+As you might guess, the publisher `example.com` is responsible for this package.
+
+`https://example.com/.well-known/packages@oauth3.org/` is the known location where package types can be discovered.
+
+Since we're using `walnut.js` which is published by daplie.com, we can find walnut packages at
+
+`https://example.com/.well-known/packages@oauth3.org/walnut.js@daplie.com.json`
+
+This file tells us where example.com publishes packages that adhere to the `walnut.js@daplie.com` package spec.
+(you can imagine that if walnut were to be implemented in ruby the ruby packages could be found at `walnut.rb@daplie.com`
+or if walnut were not protected by trademark and another company were to create a similar, but incompatible package
+system for it, it would be `walnut.go@acme.co` or some such)
+
+For publishers with a long list of packages you might find a URL to describe
+where more information about a package can be found.
+
+Template variables
+```
+:package_name
+:package_version
+```
+
+```json
+{ "package_url": "https://packages.example.com/indexes/:package_name.json"
+, "package_index": "https://packages.example.com/index.json"
+, "pingback_url": "https://api.example.com/api/pingback@oauth3.org/:package_name?version=:package_version"
+}
+```
+
+For publishers with a short list of packages you might find that all of the packages are listed directly.
+
+Template variables
+```
+:package_name
+:package_version
+:payment_token
+```
+
+```json
+{ "package_url": null
+, "package_index": null
+, "pingback_url": "https://api.example.com/api/pingback@oauth3.org/:package_name?version=:package_version"
+, "packages": [
+  { "name": "hello@example.com"
+  , "license": "Physical-Source-v2@licenses.org"
+  , "requires_payment": true
+  , "payment_url": "https://author.tld/api/payments@oauth3.org/schemas/packages/walnut.js@daplie.com/:package_name"
+  , "zip_url": "https://cdn.tld/api/orders@cdn.tld/:package_name-:package_version.zip?authorization=:payment_token"
+  , "git_https_url":"https://git.cdn.tld/author.tld/:package_name.git#:package_version?authorization=:payment_token"
+  , "git_ssh_url":":payment_token@git.cdn.tld:author.tld/:package_name.git#:package_version"
+  }
+, { "name": "gizmo@example.com"
+  , "license": "MIT@licenses.org"
+  , "requires_payment": false
+  , "zip_url": "https://example.com/packages/:package_name-:package_version.zip"
+  , "git_https_url":"https://git.cdn.tld/author.tld/:package_name.git#:package_version"
+  , "git_ssh_url":"git@git.cdn.tld:author.tld/:package_name.git#:package_version"
+  }
+] }
+```
+
+**Note**: It is not expected that the package manage will directly query the publisher -
+a centralized caching service may be used.
+However, it is intended that a package manager *could* query the publisher, even if the
+publisher points back to a centralized cdn.
+
+Package Layout
+--------------
+
+Packages have data model, api, and RESTful components.
+
+```
+/srv/walnut/packages/rest/hello@example.com/
+  package.json
+  api.js
+  models.js
+  rest.js
+```
+
+Each package must be enabled on a per-domain basis.
+
+```
+/srv/walnut/packages/client-api-grants/provider.example.com
+  '''
+  hello@example.com
+  '''
+```
+
+When a package is enabled for `example.com` it becomes immediately available via https
+as `https://api.example.com/api/package@publisher.tld/`.
+
+Note: although hot-loading of packages is supported, reloading still requires
+restarting the walnut server - for now at least
+
+Package APIs
+------------
+
+Packages are intended to be functional, however, they allow for instantiation as
+a matter of not putting ourselves in a box and finding out later that it's very,
+very, very hard to open the box back up.
+
+`rest.js`:
+```js
+module.exports.create = function (conf, deps, app) {
+  var API = require('./api.js');
+  var REST = {
+    hello: function (req, res/*, next*/) {
+      var promise = API.hello(deps, req.Models, req.oauth3/*, opts*/);
+
+      app.handlePromise(req, res, promise, "[hello@example.com]");
+    }
+  }
+};
+```
+
+### Special methods for `app`:
+
+```js
+app.handlePromise(request, response, promise, message);
+```
+
+`handlePromise` will respond to the request with the result of `promise` as JSON.
+If there is an error, it will include `message` in order to help you debug.
+
+### Special properties of `request`:
+
+```js
+req.getSiteCapability(pkg)          // Promises a capability on behalf of the current site (req.experienceId) without exposing secrets
+
+req.webhookParser(pkg, req, opts)   // Allows the use of potentially dangerous parsers (i.e. urlencoded) for the sake of webhooks
+
+req.apiUrlPrefix                    // This represents the full package path without any package specific endpoints
+                                    // This is particularly useful when constructing webhook URLs
+                                    // i.e. https://api.example.com/api/pkg@domain.tld
+                                    //      (of https://api.example.com/api/pkg@domain.tld/public/foo)
+
+req.experienceId                    // The instance name of an app as a whole, where an app is mounted
+                                    // i.e. the 'example.com' part of https://example.com/foo
+                                    //      OR 'example.com#foo' if '/foo' is part of the app's mount point
+
+req.clientApiUri                    // The api URL for the instance of an app
+                                    // i.e. the 'api.example.com' part of https://api.example.com/api/hello@example.com/kv/foo
+
+req.pkgId                           // The name of the package being accessed
+                                    // i.e. the 'hello@example.com' part of https://api.example.com/api/hello@example.com/kv/foo
+
+req.oauth3.accountIdx               // The system id of the account represented by the token
+                                    // i.e. this is the user
+```
+
+Internal (and/or deprecated) APIs that you will very likely encounter
+
+```js
+req.getSiteStore().then(function (models) {
+  req.Models = models;
+});
+
+//
+// Consider Models for a package 'hello@example.com', the would be named like so
+//
+req.Models.HelloExampleComData.create(obj)
+req.Models.ComExampleHelloData.save(obj)
+req.Models.ComExampleHelloData.find(params)
+req.Models.ComExampleHelloData.destroy(objOrId)
+
+//
+// These should be scoped in such a way that the only hand back data specific
+// to the experience and not expose secrets
+//
+
+req.getSiteConfig('com.example.hello').then(function (config) {
+    // the com.example.hello section of /srv/walnut/etc/:domain/config.json
+});
+req.getSitePackageConfig
+
+//
+// Deprecated
+//
+// These helper methods should be moved to a capability
+
+req.Stripe
+req.Mandrill
+req.Mailchimp
+
+req.getSiteMailer().then(function (mailer) {});
+```
+
+RESTful API Contstraints
+------------------------
+
+Walnut will reject requests to all domains and subdomains except those that begin with the subdomain `api`, `assets`, and `webhooks`.
+
+* `api` is for JSON APIs and must use JWT in HTTP Authorization headers for authentication
+  * secured by disallowing cookies
+  * secured by disallowing non-JSON form types
+  * secured by requiring authentication in header
+* `assets` is for protected access to large files and other blobs and must use JWT in Cookies for authentication
+  * warning: allows implicit authorization via cookies for hotlinking and the like
+  * secured by not exposing tokens when users copy-paste
+* `webhooks` is for 3rd-party API hooks and APIs with special requirements outside of the normal security model
+  * warning: these are insecure and should be used with caution, prudence, and wisdom
+  * JWT via query parameter
+  * urlencoded forms
+  * XML forms
+
+Bare and www domains are DISALLOWED from being served by Walnut.
+
+This enables scalability of static sites as the static assets
+are never on the same domain as generic APIs or authenticated assets.
+It also enforces security by disallowing 1990s web vulnerabilities by default.

CHANGELOG

@@ -0,0 +1,4 @@
+v1.2.5 - Beginning of CHANGELOG
+	* has semi-functional launchpad
+	* OAuth3 with issuer-rewrite merged in
+	* capabilities API

INSTALL.md

@@ -0,0 +1,316 @@
+From 0 to "Hello World"
+=======================
+
+Goal:
+
+The purpose of this tutorial is to install Walnut and be able to launch a simple "Hello World" app.
+
+Pre-requisites:
+
+* You have compatible server hardware
+  * Daplie Server
+  * EspressoBin
+  * Raspberry Pi
+  * MacBook
+  * (pretty much anything, actually)
+* You have compatible software
+  * Linux of any sort that uses systemd
+  * macOS using launchd
+* You own a domain
+  * through Daplie Domains
+  * or you understand domains and DNS and all that stuff
+* Install bower `npm install -g bower`
+
+Choose a domain
+---------------
+
+For the purpose of this instruction we'll assume that your domain is `foo.com`,
+but you can use, say, `johndoe.daplie.me` for testing through Daplie Domains.
+
+Anyway, go ahead and set the bash variable `$my_domain` for the purposes of the
+rest of this tutorial:
+
+```
+my_domain=foo.com
+```
+
+You can purchase a domain with daplie tools
+
+```bash
+npm install -g git+https://git.daplie.com/Daplie/daplie-tools.git
+
+daplie domains:search -n $my_domain
+```
+
+Subdomains
+----------
+
+Auth will be loaded with the following domains
+
+```
+provider.foo.com
+api.provider.foo.com
+```
+
+The Hello World app will be loaded with the following domains
+
+```
+foo.com
+www.foo.com
+api.foo.com
+assets.foo.com
+```
+
+The domains can be setup through the Daplie Desktop App or with daplie-tools
+
+Replace `foodevice` with whatever you like to call this device
+
+```bash
+# hostname
+my_device=foodevice
+
+# curl https://api.oauth3.org/api/tunnel@oauth3.org/checkip
+# READ THIS: localhost is being used as an example.
+# Your IP address should be public facing (i.e. port-forwarding is enabled on your router).
+# If it isn't, then you need something like goldilocks providing a tunnel.
+my_address=127.0.0.1
+
+# set device address and attach primary domain
+daplie devices:attach -d $my_device -n $my_domain -a $my_address
+
+# attach all other domains with same device/address
+daplie devices:attach -d $my_device -n provider.$my_domain
+daplie devices:attach -d $my_device -n api.provider.$my_domain
+daplie devices:attach -d $my_device -n www.$my_domain
+daplie devices:attach -d $my_device -n api.$my_domain
+daplie devices:attach -d $my_device -n assets.$my_domain
+daplie devices:attach -d $my_device -n cloud.$my_domain
+daplie devices:attach -d $my_device -n api.cloud.$my_domain
+```
+
+Goldilocks Configuration
+------------------------
+
+Walnut must sit behind a proxy that properly terminates https and sets the `X-Forwarded-Proto` header.
+
+Goldilocks can do this, as well as manage daplie domains, tunneling, etc.
+
+```bash
+curl https://git.daplie.com/Daplie/daplie-snippets/raw/master/install.sh | bash
+
+daplie-install-goldilocks
+```
+
+
+Example `/etc/goldilocks/goldilocks.yml`:
+```yml
+tls:
+  email: user@mailservice.com
+  servernames:
+    - foo.com
+    - www.foo.com
+    - api.foo.com
+    - assets.foo.com
+    - cloud.foo.com
+    - api.cloud.foo.com
+    - provider.foo.com
+    - api.provider.foo.com
+
+http:
+  trust_proxy: true
+  modules:
+    - name: proxy
+      domains:
+        - '*'
+      address: '127.0.0.1:3000'
+```
+
+Basic Walnut Install
+--------------------
+
+```bash
+curl https://git.daplie.com/Daplie/daplie-snippets/raw/master/install.sh | bash
+
+daplie-install-walnut
+```
+
+You could also, of course, try installing from the repository directly
+(especially if you have goldilocks or some similar already installed)
+
+```bash
+mkdir -p /srv/walnut/
+git clone https://git.daplie.com/Daplie/walnut.js.git /srv/walnut/core
+pushd /srv/walnut/core
+  git checkout v1
+popd
+bash /srv/walnut/core/install-helper.sh
+```
+
+Initial Configuration
+-------------
+
+Once installed and started you can visit <https://localhost.daplie.me:3000> to configure the primary domain.
+
+You could also do this manually via curl:
+
+```bash
+curl -X POST http://api.localhost.daplie.me:3000/api/walnut@daplie.com/init \
+  -H 'X-Forwarded-Proto: https' \
+  -H 'Content-Type: application/json' \
+  -d '{ "domain": "'$my_domain'" }'
+```
+
+Resetting the Initialization
+----------------------------
+
+Once you run the app the initialization files will appear in these locations
+
+```
+/srv/walnut/var/walnut+config@daplie.com.sqlite3
+/srv/walnut/config/foo.com.json
+```
+
+Deleting those files and restarting walnut will reset it to its bootstrap state.
+
+Reset Permissions
+-----------------
+
+Since the app store and package manager are not built yet,
+you should also change the permissions on the walnut directory for the purposes of this tutorial:
+
+```bash
+sudo chown -R $(whoami) /srv/walnut/
+sudo chmod -R +s /srv/walnut/
+```
+
+Install OAuth3 API Package
+--------------
+
+We need to have a local login system.
+
+For the APIs for that we'll install the `issuer@oauth3.org` API package and enable it for `api.provider.example.com`:
+
+```bash
+# API packaged for walnut
+git clone https://git.daplie.com/OAuth3/issuer_oauth3.org.git /srv/walnut/packages/rest/issuer@oauth3.org
+pushd /srv/walnut/packages/rest/issuer@oauth3.org/
+    git checkout v1.2
+    npm install
+popd
+
+# Give permission for this package to provider.example.com
+# the api. prefix is omitted because it is always assumed for APIs
+echo "issuer@oauth3.org" >> /srv/walnut/packages/client-api-grants/provider.$my_domain
+```
+
+*NOTE*: Currently there are some hard-coded values that need to be changed out (TODO use `getSiteConfig()`).
+`vim /srv/walnut/packages/rest/issuer@oauth3.org/lib/provide-oauth3.js` and search for the email stuff and change it.
+
+
+For the user interface for that we'll install the `issuer@oauth3.org` site package and enable it
+
+```bash
+# Frontend
+git clone https://git.daplie.com/OAuth3/org.oauth3.git /srv/walnut/packages/pages/issuer@oauth3.org
+pushd /srv/walnut/packages/pages/issuer@oauth3.org
+  bash ./install.sh
+popd
+
+# Tell Walnut to load this site package when provider.example.com is requested
+echo "issuer@oauth3.org" >> /srv/walnut/var/sites/provider.$my_domain
+```
+
+OAuth3 Secrets
+--------------
+
+OAuth3 is currently configured to use mailgun for sending verification emails.
+It is intended to provide a way to use various mail services in the future,
+just bear with us for the time being (or open a Merge Request).
+
+```bash
+mkdir -p /srv/walnut/var/provider.$my_domain
+vim /srv/walnut/var/provider.$my_domain/config.json
+```
+
+```json
+{ "mailgun.org": {
+    "apiKey": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  , "auth": {
+      "user": "robtherobot@example.com"
+    , "pass": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "api_key": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "domain": "example.com"
+    }
+  }
+, "issuer@oauth3.org": {
+    "mailer": {
+      "from": "login@example.com"
+    , "subject": "Login code request"
+    , "text": ":code\n\nis your login code"
+    }
+  }
+}
+```
+
+Install the 'hello@example.com' package
+---------------------
+
+```bash
+git clone https://git.daplie.com/Daplie/com.example.hello.git /srv/walnut/packages/rest/hello@example.com
+
+echo "hello@example.com" >> /srv/walnut/packages/client-api-grants/provider.$my_domain
+```
+
+What it should look like:
+
+```
+/srv/walnut/packages/rest/hello@example.com/
+  package.json
+  api.js
+  models.js
+  rest.js
+
+/srv/walnut/packages/client-api-grants/provider.foo.com
+  '''
+  issuer@oauth3.org
+  hello@example.com
+  '''
+```
+
+Setup the Seed App (front-end)
+------------------------
+
+Get the Seed App
+
+```bash
+pushd /srv/walnut/packages/pages/
+
+git clone https://git.daplie.com/Daplie/seed_example.com.git --branch v1 seed@example.com
+
+pushd seed@example.com/
+  git clone https://git.daplie.com/OAuth3/oauth3.js.git --branch v1.1 assets/oauth3.org
+
+  mkdir -p .well-known
+  ln -sf  ../assets/oauth3.org/.well-known/oauth3 .well-known/oauth3
+popd
+
+echo "seed@example.com" >> /srv/walnut/var/sites/$my_domain
+
+popd
+```
+
+You will need to change the authenication provider/issuer URL from `oauth3.org` to the domain you've selected (i.e. `provider.example.com`)
+
+```bash
+vim /srv/walnut/packages/pages/seed@example.com/js/config.js
+```
+
+```js
+{ "azp@oauth3.org": { issuer_uri: 'provider.example.com', client_uri: 'example.com' } }
+```
+
+See Hello World
+---------------
+
+Now visit your site (i.e. https://example.com) and you will be able to login
+and access the hello world data.

LICENSE

@@ -1,182 +1,32 @@
-Apache License
+Copyright 2017 Daplie, Inc
-Version 2.0, January 2004
-http://www.apache.org/licenses/
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+This is open source software; you can redistribute it and/or modify it under the
+terms of either:
 
-1. Definitions.
+   a) the "MIT License"
+   b) the "Apache-2.0 License"
 
-"License" shall mean the terms and conditions for use, reproduction, and
+MIT License
-distribution as defined by Sections 1 through 9 of this document.
 
-"Licensor" shall mean the copyright owner or entity authorized by the copyright
+   Permission is hereby granted, free of charge, to any person obtaining a copy
-owner that is granting the License.
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
 
-"Legal Entity" shall mean the union of the acting entity and all other entities
+   The above copyright notice and this permission notice shall be included in all
-that control, are controlled by, or are under common control with that entity.
+   copies or substantial portions of the Software.
-For the purposes of this definition, "control" means (i) the power, direct or
-indirect, to cause the direction or management of such entity, whether by
-contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
-outstanding shares, or (iii) beneficial ownership of such entity.
 
-"You" (or "Your") shall mean an individual or Legal Entity exercising
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-permissions granted by this License.
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
 
-"Source" form shall mean the preferred form for making modifications, including
+Apache-2.0 License Summary
-but not limited to software source code, documentation source, and configuration
-files.
-
-"Object" form shall mean any form resulting from mechanical transformation or
-translation of a Source form, including but not limited to compiled object code,
-generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made
-available under the License, as indicated by a copyright notice that is included
-in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that
-is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative Works
-shall not include works that remain separable from, or merely link (or bind by
-name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative Works
-thereof, that is intentionally submitted to Licensor for inclusion in the Work
-by the copyright owner or by an individual or Legal Entity authorized to submit
-on behalf of the copyright owner. For the purposes of this definition,
-"submitted" means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor for
-the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently
-incorporated within the Work.
-
-2. Grant of Copyright License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the Work and such
-Derivative Works in Source or Object form.
-
-3. Grant of Patent License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable (except as stated in this section) patent license to make, have
-made, use, offer to sell, sell, import, and otherwise transfer the Work, where
-such license applies only to those patent claims licensable by such Contributor
-that are necessarily infringed by their Contribution(s) alone or by combination
-of their Contribution(s) with the Work to which such Contribution(s) was
-submitted. If You institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work or a
-Contribution incorporated within the Work constitutes direct or contributory
-patent infringement, then any patent licenses granted to You under this License
-for that Work shall terminate as of the date such litigation is filed.
-
-4. Redistribution.
-
-You may reproduce and distribute copies of the Work or Derivative Works thereof
-in any medium, with or without modifications, and in Source or Object form,
-provided that You meet the following conditions:
-
-You must give any other recipients of the Work or Derivative Works a copy of
-this License; and
-You must cause any modified files to carry prominent notices stating that You
-changed the files; and
-You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source form
-of the Work, excluding those notices that do not pertain to any part of the
-Derivative Works; and
-If the Work includes a "NOTICE" text file as part of its distribution, then any
-Derivative Works that You distribute must include a readable copy of the
-attribution notices contained within such NOTICE file, excluding those notices
-that do not pertain to any part of the Derivative Works, in at least one of the
-following places: within a NOTICE text file distributed as part of the
-Derivative Works; within the Source form or documentation, if provided along
-with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents of
-the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works that
-You distribute, alongside or as an addendum to the NOTICE text from the Work,
-provided that such additional attribution notices cannot be construed as
-modifying the License.
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction, or
-distribution of Your modifications, or for any such Derivative Works as a whole,
-provided Your use, reproduction, and distribution of the Work otherwise complies
-with the conditions stated in this License.
-
-5. Submission of Contributions.
-
-Unless You explicitly state otherwise, any Contribution intentionally submitted
-for inclusion in the Work by You to the Licensor shall be under the terms and
-conditions of this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify the terms of
-any separate license agreement you may have executed with Licensor regarding
-such Contributions.
-
-6. Trademarks.
-
-This License does not grant permission to use the trade names, trademarks,
-service marks, or product names of the Licensor, except as required for
-reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty.
-
-Unless required by applicable law or agreed to in writing, Licensor provides the
-Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE,
-NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
-solely responsible for determining the appropriateness of using or
-redistributing the Work and assume any risks associated with Your exercise of
-permissions under this License.
-
-8. Limitation of Liability.
-
-In no event and under no legal theory, whether in tort (including negligence),
-contract, or otherwise, unless required by applicable law (such as deliberate
-and grossly negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special, incidental,
-or consequential damages of any character arising as a result of this License or
-out of the use or inability to use the Work (including but not limited to
-damages for loss of goodwill, work stoppage, computer failure or malfunction, or
-any and all other commercial damages or losses), even if such Contributor has
-been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability.
-
-While redistributing the Work or Derivative Works thereof, You may choose to
-offer, and charge a fee for, acceptance of support, warranty, indemnity, or
-other liability obligations and/or rights consistent with this License. However,
-in accepting such obligations, You may act only on Your own behalf and on Your
-sole responsibility, not on behalf of any other Contributor, and only if You
-agree to indemnify, defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason of your
-accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets "[]" replaced with your own
-identifying information. (Don't include the brackets!) The text should be
-enclosed in the appropriate comment syntax for the file format. We also
-recommend that a file or class name and description of purpose be included on
-the same "printed page" as the copyright notice for easier identification within
-third-party archives.
-
-   Copyright 2013 AJ ONeal
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,61 +1,110 @@
 walnut
 ======
 
-Small, light, and secure iot application framework.
+An opinionated, constrained, secure application framework with a hard shell - kinda like iOS, but for a server.
 
-```bash
+Applications are written in express, but instead of using `require` for generic packages,
-curl https://git.daplie.com/Daplie/daplie-snippets/raw/master/install.sh | bash
+they use `req.getSiteCapability(pkg)` and are restricted to packages that have been
+allowed by app, device, site, or user permission. Any configuration for the capability
+(external passwords, api keys, etc) will be set up beforehand so that they are not exposed
+to the application.
 
-daplie-install-cloud
+Security Features
-```
+-----------------
 
-Features
+* JSON-only APIs
-------
+* JWT (not cookie*) authentication
-
+* no server-rendered html
-* Works with Goldilocks for secure, Let's Encrypt maneged, https-only serving
+* disallows urlencoded forms, except for secured webhooks
-
+* disallows cookies, except for protected static assets
-* IOT Application server written in [Node.js](https://nodejs.org)
+* api.* subdomain for apis
-* Small memory footprint (for a node app)
+* assets.* subdomain for protected assets
-* Secure
+* *must* sit behind a trusted https proxy (such as [Goldilocks](https://git.coolaj86.com/coolaj86/goldilocks.js))
-  * Uses JWT, not Cookies\*
 * HTTPS-only (checks for X-Forwarded-For)
 * AES, RSA, and ECDSA encryption and signing
 * Safe against CSRF, XSS, and SQL injection
 * Safe against Compression attacks
-* Multi-Tentated Application Management
-* Built-in OAuth2 & OAuth3 support
 
-\*Cookies are used only for GETs and only where using a token would be less secure
+\*Cookies are used only for GETs and only where using a token would be less secure -
 such as images which would otherwise require the token to be passed into the img src.
 They are also scoped such that CSRF attacks are not possible.
 
-In Progress
+Application Features
------------
+--------------------
 
-* HTTPS Key Pinning
+* JSON-only expressjs APIs
-* Heroku (pending completion of PostgreSQL support)
+* Capability-based permissions system for (oauth3-discoverable) packages such as
-* [GunDB](https://gundb.io) Support
+  * large file access (files@oauth3.org)
-* OpenID support
+  * database access (data@oauth3.org)
-
+  * scheduling (for background tasks, alerts, alarms, calendars, reminders, etc) (events@oauth3.org)
-Structure
+  * payments (credit card) (payments@oauth3.org)
-=====
+  * email (email@oauth3.org)
+  * SMS (texting) (tel@oauth3.org)
+  * voice (calls and answering machine) (tel@oauth3.org)
+  * lamba-style functions (functions@oauth3.org)
+* Per-app, per-site, and per-user configurations
+* Multi-Tentated Application Management
+* Built-in OAuth2 & OAuth3 support
 
 Currently being tested with Ubuntu, Raspbian, and Debian on Digital Ocean, Raspberry Pi, and Heroku.
 
+Installation
+------------
+
+We're still in a stage where the installation generally requires many manual steps.
+
+```bash
+curl https://git.coolaj86.com/coolaj86/walnut.js/raw/v1.2/installer/get.sh | bash
+```
+
+See [INSTALL.md](/INSTALL.md)
+
+### Uninstall
+
+```bash
+rm -rf /srv/walnut/ /var/walnut/ /etc/walnut/ /opt/walnut/ /var/log/walnut/ /etc/systemd/system/walnut.service /etc/tmpfiles.d/walnut.conf
+```
+
+Usage
+-----
+
+Here's how you run the thing, once installed:
+
+```
+/opt/walnut/bin/node /srv/walnut/core/bin/walnut.js
+```
+
+It listens on all addresses, port 3000.
+
+TODO: Add config to restrict listening to localhost.
+
+API
+---
+
+The API is still in flux, but you can take a peek anyway.
+
+See [API.md](/API.md)
+
+Understanding Walnut
+====================
+
 ```
 /srv/walnut/
 ├── setup.sh (in-progress)
 ├── core
 │   ├── bin
 │   ├── boot
-│   ├── holepunch
 │   └── lib
+├── etc
+│   └── client-api-grants
 ├── node_modules
 ├── packages
 │   ├── apis
 │   ├── pages
+│   ├── rest
 │   └── services
 └── var
+    └── sites
 ```
 
 * `core` contains all walnut code
@@ -76,31 +125,46 @@ Will install to
 /etc/tmpfiles.d/walnut.conf
 ```
 
-Implementation details
-----------------
-
 Initialization
 --------------
 
 needs to know its primary domain
 
 ```
-POST https://api.<domain.tld>/api/com.daplie.walnut.init
+POST https://api.<domain.tld>/api/walnut@oauth3.org/init
 
 { "domain": "<domain.tld>" }
 ```
 
 The following domains are required to point to WALNUT server
 
+```
+cloud.<domain.tld>
+api.cloud.<domain.tld>
+```
+
+and
+
 ```
 <domain.tld>
 www.<domain.tld>
 
 api.<domain.tld>
 assets.<domain.tld>
+```
 
-cloud.<domain.tld>
+The domains can be setup through the OAuth3 Desktop App or with `oauth3-tools`
-api.cloud.<domain.tld>
+
+```bash
+# set device address and attach primary domain
+oauth3 devices:attach -d foodevice -n example.com -a 127.0.0.1
+
+# attach all other domains with same device/address
+oauth3 devices:attach -d foodevice -n www.example.com
+oauth3 devices:attach -d foodevice -n api.example.com
+oauth3 devices:attach -d foodevice -n assets.example.com
+oauth3 devices:attach -d foodevice -n cloud.example.com
+oauth3 devices:attach -d foodevice -n api.cloud.example.com
 ```
 
 Example `/etc/goldilocks/goldilocks.yml`:
@@ -130,11 +194,11 @@ Resetting the Initialization
 Once you run the app the initialization files will appear in these locations
 
 ```
-/srv/walnut/var/com.daplie.walnut.config.sqlite3
+/srv/walnut/var/walnut+config@oauth3.org.sqlite3
 /srv/walnut/config/<domain.tld>/config.json
 ```
 
-Deleting those files will rese
+Deleting those files and restarting walnut will reset it to its bootstrap state.
 
 Accessing static apps
 ---------------------
@@ -144,11 +208,11 @@ Static apps are stored in `packages/pages`
 ```
 # App ID as files with a list of packages they should load
 # note that '#' is used in place of '/' because files and folders may not contain '/' in their names
-/srv/walnut/packages/sites/<domain.tld#path>          # https://domain.tld/path
+/srv/walnut/packages/pages/<domain.tld#path>          # https://domain.tld/path
-/srv/walnut/packages/sites/<domain.tld>               # https://domain.tld and https://domain.tld/foo match
+/srv/walnut/packages/pages/<domain.tld>               # https://domain.tld and https://domain.tld/foo match
 
-# packages are directories with reverse dns name      # For the sake of debugging these packages can be accessed directly, without a site by
+# packages are directories with email-style name      # For the sake of debugging these packages can be accessed directly, without a site by
-/srv/walnut/packages/pages/<tld.domain.package>       # matches apps.<domain.tld>/<package-name> and <domain.tld>/apps/<package-name>
+/srv/walnut/packages/pages/<package@domain.tld>       # matches apps.<domain.tld>/<package-name> and <domain.tld>/apps/<package-name>
 ```
 
 Accessing REST APIs
@@ -174,18 +238,6 @@ The packages:
 ```
 /srv/walnut/packages/
 ├── api
-├── pages
-│   └── com.example.hello
-│       └── index.html
-│             '''
-│             <html>
-│               <head><title>com.example.hello</title></head>
-│               <body>
-│                 <h1>com.example.hello</h1>
-│               </body>
-│             </html>
-│             '''
-│
 ├── rest
 │   └── com.example.hello
 │      ├── package.json
@@ -208,19 +260,38 @@ The packages:
 └── services
 ```
 
+```
+/srv/walnut/packages/
+└── pages
+    └── demo@example.com
+        └── index.html
+              '''
+              <html>
+                <head><title>demo@example.com</title></head>
+                <body>
+                  <h1>demo@example.com</h1>
+                </body>
+              </html>
+              '''
+
+```
+
 The permissions:
 
 ```
 /srv/walnut/packages/
-├── client-api-grants
+└── client-api-grants
-│   └── cloud.foobar.me
+    └── cloud.foobar.me
-│         '''
-│         com.example.hello     # refers to /srv/walnut/packages/rest/com.example.hello
-│         '''
-│
-└── sites
-    └── daplie.me
           '''
-          com.example.hello     # refers to /srv/walnut/packages/pages/com.example.hello
+          hello@example.com     # refers to /srv/walnut/packages/rest/hello@example.com
+          '''
+```
+
+```
+/srv/walnut/var/
+└── sites
+    └── example.com
+          '''
+          seed@example.com      # refers to /srv/walnut/packages/pages/seed@example.com
           '''
 ```

bin/walnut.js

@@ -2,11 +2,6 @@
 'use strict';
 
 require('../walnut.js');
-/*
-var c = require('console-plus');
-console.log = c.log;
-console.error = c.error;
-*/
 
 function eagerLoad() {
   var PromiseA = require('bluebird').Promise;

boot/master.js

@@ -82,13 +82,8 @@ cluster.on('online', function (worker) {
 cluster.on('exit', function (worker, code, signal) {
   console.info('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
 
-  workers = workers.map(function (w) {
+  workers = workers.filter(function (w) {
-    if (worker !== w) {
+    return w && w !== worker;
-      return w;
-    }
-    return null;
-  }).filter(function (w) {
-    return w;
   });
 
   //console.log('WARNING: worker spawning turned off for debugging ');

boot/worker.js

@@ -149,9 +149,10 @@ module.exports.create = function () {
   process.on('unhandledRejection', function (err) {
     // this should always throw
     // (it means somewhere we're not using bluebird by accident)
-    console.error('[caught] [unhandledRejection]');
+    console.error('[caught unhandledRejection]:', err.message || '');
-    console.error(Object.keys(err));
+    Object.keys(err).forEach(function (key) {
-    console.error(err);
+      console.log('\t'+key+': '+err[key]);
+    });
     console.error(err.stack);
   });
   process.on('rejectionHandled', function (msg) {

dist/etc/systemd/system/walnut.service

@@ -19,15 +19,15 @@ StartLimitBurst=3
 
 # User and group the process will run as
 # (www-data is the de facto standard on most systems)
-User=www-data
+User=MY_USER
-Group=www-data
+Group=MY_GROUP
 
 # If we need to pass environment variables in the future
 ; Environment=GOLDILOCKS_PATH=/opt/walnut
 
 # Set a sane working directory, sane flags, and specify how to reload the config file
-WorkingDirectory=/srv/www
+WorkingDirectory=/opt/walnut
-ExecStart=/usr/local/bin/node /srv/walnut/core/bin/walnut.js --config=/etc/walnut/walnut.yml
+ExecStart=/opt/walnut/bin/node /opt/walnut/core/bin/walnut.js --config=/etc/walnut/walnut.yml
 ExecReload=/bin/kill -USR1 $MAINPID
 
 # Limit the number of file descriptors and processes; see `man systemd.exec` for more limit settings.
@@ -46,7 +46,7 @@ ProtectSystem=full
 # … except TLS/SSL, ACME, and Let's Encrypt certificates
 #   and /var/log/, because we want a place where logs can go.
 #   This merely retains r/w access rights, it does not add any new. Must still be writable on the host!
-ReadWriteDirectories=/etc/walnut /var/log/walnut /var/walnut /opt/walnut /srv/www
+ReadWriteDirectories=/etc/walnut /var/log/walnut /var/walnut /opt/walnut /srv/walnut
 
 # Note: in v231 and above ReadWritePaths has been renamed to ReadWriteDirectories
 ; ReadWritePaths=/etc/walnut /var/log/walnut

dist/etc/tmpfiles.d/walnut.conf

@@ -1,12 +1,5 @@
-# /etc/tmpfiles.d/walnut.conf
+# /etc/tmpfiles.d/goldilocks.conf
 # See https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html
 
 # Type Path           Mode UID      GID      Age Argument
-d /etc/walnut          0755 www-data www-data -   -
+d /run/goldilocks          0755 MY_USER MY_GROUP -   -
-d /etc/ssl/walnut      0750 www-data www-data -   -
-d /srv/walnut          0775 www-data www-data -   -
-d /srv/www             0775 www-data www-data -   -
-d /opt/walnut          0775 www-data www-data -   -
-d /var/walnut          0775 www-data www-data -   -
-d /var/log/walnut      0750 www-data www-data -   -
-#d /run/walnut          0755 www-data www-data -   -

dist/srv/walnut/var/example.com/config.json

@@ -0,0 +1,56 @@
+{
+  "com.daplie.domains.enom": {
+    "live": {
+        "url": "https://reseller.enom.com/interface.asp"
+      , "baseUrl": "https://reseller.enom.com/interface.asp"
+      , "id": "xxxxxxxx"
+      , "secret": "xxxxxxxxxxxxxxxx"
+    }
+    , "test": {
+        "url": "https://resellertest.enom.com/interface.asp"
+      , "baseUrl": "https://resellertest.enom.com/interface.asp"
+      , "id": "xxxxxxxx"
+      , "secret": "xxxxxxxxxxxxxxxx"
+    }
+  }
+, "mailgun.org": {
+    "apiKey": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  , "apiPublicKey": "pubkey-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  , "auth": {
+      "user": "robtherobot@example.com"
+    , "pass": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "api_key": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "domain": "example.com"
+    }
+  }
+, "mailchimp.com": {
+    "apiKey": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-usxx"
+  }
+, "mandrill.com": {
+    "apiKey": "xxxxxxxxxxxxxxxxxxxxxx"
+	}
+, "stripe.com": {
+	  "live": {
+			"id": "pk_live_xxxxxxxxxxxxxxxxxxxxxxxx"
+		, "secret": "sk_live_xxxxxxxxxxxxxxxxxxxxxxxx"
+		}
+	, "test": {
+			"id": "pk_test_xxxxxxxxxxxxxxxxxxxxxxxx"
+		, "secret": "sk_test_xxxxxxxxxxxxxxxxxxxxxxxx"
+		}
+  }
+, "com.daplie.ddns": {
+    "privKeyPem": "-----BEGIN RSA PRIVATE KEY-----\nXXXXXXXX==\n-----END RSA PRIVATE KEY-----"
+  }
+, "org.oauth3.tunnel": {
+    "live": {
+      "secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "url": "wss://tunnel.example.com/"
+    }
+  , "test": {
+      "secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    , "url": "wss://test-tunnel.example.com/"
+    , "dnsUrl": ""
+    }
+  }
+}

install.sh

@@ -1,259 +0,0 @@
-#!/bin/bash
-
-set -e
-set -u
-
-# something or other about android and tmux using PREFIX
-#: "${PREFIX:=''}"
-MY_ROOT=""
-if [ -z "${PREFIX-}" ]; then
-  MY_ROOT=""
-else
-  MY_ROOT="$PREFIX"
-fi
-# Not every platform has or needs sudo, gotta save them O(1)s...
-sudo_cmd=""
-((EUID)) && [[ -z "${ANDROID_ROOT-}" ]] && sudo_cmd="sudo"
-
-###############################
-#                             #
-#         http_get            #
-# boilerplate for curl / wget #
-#                             #
-###############################
-
-# See https://git.daplie.com/Daplie/daplie-snippets/blob/master/bash/http-get.sh
-
-http_curl_opts="-fsSL"
-http_wget_opts="--quiet"
-
-http_bin=""
-http_opts=""
-http_out=""
-
-detect_http_bin()
-{
-  if type -p curl >/dev/null 2>&1; then
-    http_bin="curl"
-    http_opts="$http_curl_opts"
-    http_out="-o"
-    #curl -fsSL "$url" -o "$PREFIX/tmp/$pkg"
-  elif type -p wget >/dev/null 2>&1; then
-    http_bin="wget"
-    http_opts="$http_wget_opts"
-    http_out="-O"
-    #wget --quiet "$url" -O "$PREFIX/tmp/$pkg"
-  else
-    echo "Aborted, could not find curl or wget"
-    return 7
-  fi
-}
-
-http_get()
-{
-  if [ -e "$1" ]; then
-    rsync -a "$1" "$2"
-  elif type -p curl >/dev/null 2>&1; then
-    $http_bin $http_curl_opts $http_out "$2" "$1"
-  elif type -p wget >/dev/null 2>&1; then
-    $http_bin $http_wget_opts $http_out "$2" "$1"
-  else
-    echo "Aborted, could not find curl or wget"
-    return 7
-  fi
-}
-
-dap_dl()
-{
-  http_get "$1" "$2"
-}
-
-dap_dl_bash()
-{
-  dap_url=$1
-  #dap_args=$2
-  rm -rf /tmp/dap-tmp-runner.sh
-  $http_bin $http_opts $http_out /tmp/dap-tmp-runner.sh "$dap_url"; bash /tmp/dap-tmp-runner.sh; rm /tmp/dap-tmp-runner.sh
-}
-
-detect_http_bin
-
-## END HTTP_GET ##
-
-
-
-###################
-#                 #
-# Install service #
-#                 #
-###################
-
-install_for_systemd()
-{
-  echo ""
-  echo "Installing as systemd service"
-  echo ""
-  mkdir -p $(dirname "$my_app_dir/$my_app_systemd_service")
-  dap_dl "$installer_base/$my_app_systemd_service" "$my_app_dir/$my_app_systemd_service"
-  $sudo_cmd mv "$my_app_dir/$my_app_systemd_service" "$MY_ROOT/$my_app_systemd_service"
-  $sudo_cmd chown -R root:root "$MY_ROOT/$my_app_systemd_service"
-  $sudo_cmd chmod 644 "$MY_ROOT/$my_app_systemd_service"
-
-  mkdir -p $(dirname "$my_app_dir/$my_app_systemd_tmpfiles")
-  dap_dl "$installer_base/$my_app_systemd_tmpfiles" "$my_app_dir/$my_app_systemd_tmpfiles"
-  $sudo_cmd mv "$my_app_dir/$my_app_systemd_tmpfiles" "$MY_ROOT/$my_app_systemd_tmpfiles"
-  $sudo_cmd chown -R root:root "$MY_ROOT/$my_app_systemd_tmpfiles"
-  $sudo_cmd chmod 644 "$MY_ROOT/$my_app_systemd_tmpfiles"
-
-  $sudo_cmd systemctl stop "${my_app_name}.service" >/dev/null 2>/dev/null
-  $sudo_cmd systemctl daemon-reload
-  $sudo_cmd systemctl start "${my_app_name}.service"
-  $sudo_cmd systemctl enable "${my_app_name}.service"
-
-  echo "$my_app_name started with systemctl, check its status like so"
-  echo "  $sudo_cmd systemctl status $my_app_name"
-  echo "  $sudo_cmd journalctl -xe -u $my_app_name"
-}
-
-install_for_launchd()
-{
-  echo ""
-  echo "Installing as launchd service"
-  echo ""
-  # See http://www.launchd.info/
-  mkdir -p $(dirname "$my_app_dir/$my_app_launchd_service")
-  dap_dl "$installer_base/$my_app_launchd_service" "$my_app_dir/$my_app_launchd_service"
-  $sudo_cmd mv "$my_app_dir/$my_app_launchd_service" "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd chown root:wheel "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd chmod 0644 "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd launchctl unload -w "$MY_ROOT/$my_app_launchd_service" >/dev/null 2>/dev/null
-  $sudo_cmd launchctl load -w "$MY_ROOT/$my_app_launchd_service"
-
-  echo "$my_app_name started with launchd"
-}
-
-install_etc_config()
-{
-  #echo "install etc config $MY_ROOT / $my_app_etc_config"
-  if [ ! -e "$MY_ROOT/$my_app_etc_config" ]; then
-    $sudo_cmd mkdir -p $(dirname "$MY_ROOT/$my_app_etc_config")
-    mkdir -p $(dirname "$my_app_dir/$my_app_etc_config")
-    dap_dl "$installer_base/$my_app_etc_config" "$my_app_dir/$my_app_etc_config"
-    $sudo_cmd mv "$my_app_dir/$my_app_etc_config" "$MY_ROOT/$my_app_etc_config"
-  fi
-
-  $sudo_cmd chown -R www-data:www-data $(dirname "$MY_ROOT/$my_app_etc_config") || true
-  $sudo_cmd chown -R _www:_www $(dirname "$MY_ROOT/$my_app_etc_config") || true
-  $sudo_cmd chmod 775 $(dirname "$MY_ROOT/$my_app_etc_config")
-  $sudo_cmd chmod 664 "$MY_ROOT/$my_app_etc_config"
-}
-
-install_service()
-{
-  install_etc_config
-  #echo "install service"
-
-  installable=""
-  if [ -d "$MY_ROOT/etc/systemd/system" ]; then
-    install_for_systemd
-    installable="true"
-  fi
-  if [ -d "/Library/LaunchDaemons" ]; then
-    install_for_launchd
-    installable="true"
-  fi
-  if [ -z "$installable" ]; then
-    echo ""
-    echo "Unknown system service init type. You must install as a system service manually."
-    echo '(please file a bug with the output of "uname -a")'
-    echo ""
-  fi
-  echo ""
-}
-
-## END SERVICE_INSTALL ##
-
-# Create dirs, set perms
-create_skeleton()
-{
-  $sudo_cmd mkdir -p /srv/www
-  $sudo_cmd mkdir -p /var/log/$my_app_name
-  $sudo_cmd mkdir -p /etc/$my_app_name
-  $sudo_cmd mkdir -p /var/$my_app_name
-  $sudo_cmd mkdir -p /srv/$my_app_name
-  $sudo_cmd mkdir -p /opt/$my_app_name
-}
-
-# Unistall
-install_uninstaller()
-{
-  #echo "install uninstaller"
-  dap_dl "https://git.daplie.com/Daplie/walnut.js/raw/master/uninstall.sh" "./walnut-uninstall"
-  $sudo_cmd chmod 755 "./walnut-uninstall"
-  $sudo_cmd chown root:root "./walnut-uninstall"
-  $sudo_cmd mv "./walnut-uninstall" "/usr/local/bin/uninstall-walnut"
-}
-
-
-# Dependencies
-dap_dl_bash "https://git.daplie.com/coolaj86/node-install-script/raw/master/setup-min.sh"
-
-# Install
-# npm install -g 'git+https://git@git.daplie.com/Daplie/walnut.js.git#fs-nosql'
-
-my_app_name=walnut
-my_app_pkg_name=com.daplie.walnut.web
-my_app_dir=$(mktemp -d)
-#installer_base="https://git.daplie.com/Daplie/walnut.js/raw/master/dist"
-#installer_base="$( dirname "${BASH_SOURCE[0]}" )/dist"
-installer_base="/srv/walnut/core/dist"
-
-my_app_etc_config="etc/${my_app_name}/${my_app_name}.yml"
-my_app_systemd_service="etc/systemd/system/${my_app_name}.service"
-my_app_systemd_tmpfiles="etc/tmpfiles.d/${my_app_name}.conf"
-my_app_launchd_service="Library/LaunchDaemons/${my_app_pkg_name}.plist"
-
-# Install
-install_my_app()
-{
-  #git clone git@git.daplie.com:Daplie/walnut.js.git
-  #git clone https://git.daplie.com/Daplie/walnut.js.git /srv/walnut/core
-  sudo mkdir -p /srv/walnut/{core,lib,var,etc,node_modules}
-  rm -rf /srv/walnut/core/node_modules
-  ln -sf ../node_modules /srv/walnut/core/node_modules
-  sudo mkdir -p /srv/walnut/var/sites
-  sudo mkdir -p /srv/walnut/etc/org.oauth3.consumer
-  sudo mkdir -p /srv/walnut/etc/org.oauth3.provider
-  sudo mkdir -p /srv/walnut/etc/client-api-grants
-  sudo mkdir -p /srv/walnut/packages/{rest,api,pages,services}
-
-  # backwards compat
-  if [ -d /srv/walnut/packages/client-api-grants ]; then
-    mv /srv/walnut/packages/client-api-grants/* /srv/walnut/etc/client-api-grants/
-    rm -r /srv/walnut/packages/client-api-grants
-    #rmdir /srv/walnut/packages/client-api-grants
-  fi
-  if [ -d /srv/walnut/packages/sites ]; then
-    mv /srv/walnut/packages/sites/* /srv/walnut/var/sites
-    rm -r /srv/walnut/packages/sites
-    #rmdir /srv/walnut/packages/sites
-  fi
-  ln -s /srv/walnut/etc/client-api-grants /srv/walnut/packages/client-api-grants
-  ln -s /srv/walnut/var/sites /srv/walnut/packages/client-api-grants
-
-  pushd /srv/walnut/core
-    npm install
-  popd
-}
-
-sudo mkdir -p /srv/walnut
-sudo chown -R $(whoami) /srv/walnut
-
-install_my_app
-create_skeleton
-install_uninstaller
-install_service
-
-sudo chown -R www-data:www-data /srv/walnut || true
-sudo chown -R _www:_www /srv/walnut || true
-sudo chmod -R ug+rwX /srv/walnut

installer/get.sh

@@ -0,0 +1,20 @@
+set -e
+set -u
+
+my_name=walnut
+# TODO provide an option to supply my_ver and my_tmp
+my_ver=master
+my_tmp=$(mktemp -d)
+
+mkdir -p $my_tmp/opt/$my_name/lib/node_modules/$my_name
+git clone https://git.coolaj86.com/coolaj86/walnut.js.git $my_tmp/opt/$my_name/core
+
+echo "Installing to $my_tmp (will be moved after install)"
+pushd $my_tmp/opt/$my_name/core
+  git checkout $my_ver
+  source ./installer/install.sh
+popd
+
+echo "Installation successful, now cleaning up $my_tmp ..."
+rm -rf $my_tmp
+echo "Done"

installer/http-get.sh

@@ -0,0 +1,48 @@
+###############################
+#                             #
+#         http_get            #
+# boilerplate for curl / wget #
+#                             #
+###############################
+
+# See https://git.coolaj86.com/coolaj86/snippets/blob/master/bash/http-get.sh
+
+_h_http_get=""
+_h_http_opts=""
+_h_http_out=""
+
+detect_http_get()
+{
+  set +e
+  if type -p curl >/dev/null 2>&1; then
+    _h_http_get="curl"
+    _h_http_opts="-fsSL"
+    _h_http_out="-o"
+  elif type -p wget >/dev/null 2>&1; then
+    _h_http_get="wget"
+    _h_http_opts="--quiet"
+    _h_http_out="-O"
+  else
+    echo "Aborted, could not find curl or wget"
+    return 7
+  fi
+  set -e
+}
+
+http_get()
+{
+  $_h_http_get $_h_http_opts $_h_http_out "$2" "$1"
+  touch "$2"
+}
+
+http_bash()
+{
+  _http_url=$1
+  #dap_args=$2
+  rm -rf dap-tmp-runner.sh
+  $_h_http_get $_h_http_opts $_h_http_out dap-tmp-runner.sh "$_http_url"; bash dap-tmp-runner.sh; rm dap-tmp-runner.sh
+}
+
+detect_http_get
+
+## END HTTP_GET ##

installer/install-for-launchd.sh

@@ -0,0 +1,17 @@
+set -u
+
+my_app_launchd_service="Library/LaunchDaemons/${my_app_pkg_name}.plist"
+
+echo ""
+echo "Installing as launchd service"
+echo ""
+
+# See http://www.launchd.info/
+safe_copy_config "$my_app_dist/$my_app_launchd_service" "$my_root/$my_app_launchd_service"
+
+$sudo_cmd chown root:wheel "$my_root/$my_app_launchd_service"
+
+$sudo_cmd launchctl unload -w "$my_root/$my_app_launchd_service" >/dev/null 2>/dev/null
+$sudo_cmd launchctl load -w "$my_root/$my_app_launchd_service"
+
+echo "$my_app_name started with launchd"

installer/install-for-systemd.sh

@@ -0,0 +1,35 @@
+set -u
+
+my_app_systemd_service="etc/systemd/system/${my_app_name}.service"
+my_app_systemd_tmpfiles="etc/tmpfiles.d/${my_app_name}.conf"
+
+echo ""
+echo "Installing as systemd service"
+echo ""
+
+sed "s/MY_USER/$my_user/g" "$my_app_dist/$my_app_systemd_service" > "$my_app_dist/$my_app_systemd_service.2"
+sed "s/MY_GROUP/$my_group/g" "$my_app_dist/$my_app_systemd_service.2" > "$my_app_dist/$my_app_systemd_service"
+rm "$my_app_dist/$my_app_systemd_service.2"
+safe_copy_config "$my_app_dist/$my_app_systemd_service" "$my_root/$my_app_systemd_service"
+
+sed "s/MY_USER/$my_user/g" "$my_app_dist/$my_app_systemd_tmpfiles" > "$my_app_dist/$my_app_systemd_tmpfiles.2"
+sed "s/MY_GROUP/$my_group/g" "$my_app_dist/$my_app_systemd_tmpfiles.2" > "$my_app_dist/$my_app_systemd_tmpfiles"
+rm "$my_app_dist/$my_app_systemd_tmpfiles.2"
+safe_copy_config "$my_app_dist/$my_app_systemd_tmpfiles" "$my_root/$my_app_systemd_tmpfiles"
+
+$sudo_cmd systemctl stop "${my_app_name}.service" >/dev/null 2>/dev/null || true
+$sudo_cmd systemctl daemon-reload
+$sudo_cmd systemctl start "${my_app_name}.service"
+$sudo_cmd systemctl enable "${my_app_name}.service"
+
+echo ""
+echo ""
+echo "Fun systemd commands to remember:"
+echo "  $sudo_cmd systemctl daemon-reload"
+echo "  $sudo_cmd systemctl restart $my_app_name.service"
+echo ""
+echo "$my_app_name started with systemctl, check its status like so:"
+echo "  $sudo_cmd systemctl status $my_app_name"
+echo "  $sudo_cmd journalctl -xefu $my_app_name"
+echo ""
+echo ""

installer/install-system-service.sh

@@ -0,0 +1,37 @@
+safe_copy_config()
+{
+  src=$1
+  dst=$2
+  $sudo_cmd mkdir -p $(dirname "$dst")
+  if [ -f "$dst" ]; then
+    $sudo_cmd rsync -a "$src" "$dst.latest"
+    # TODO edit config file with $my_user and $my_group
+    if [ "$(cat $dst)" == "$(cat $dst.latest)" ]; then
+      $sudo_cmd rm $dst.latest
+    else
+      echo "MANUAL INTERVENTION REQUIRED: check the systemd script update and manually decide what you want to do"
+      echo "diff $dst $dst.latest"
+      $sudo_cmd chown -R root:root "$dst.latest"
+    fi
+  else
+    $sudo_cmd rsync -a --ignore-existing "$src" "$dst"
+  fi
+  $sudo_cmd chown -R root:root "$dst"
+  $sudo_cmd chmod 644 "$dst"
+}
+
+installable=""
+if [ -d "$my_root/etc/systemd/system" ]; then
+  source ./installer/install-for-systemd.sh
+  installable="true"
+fi
+if [ -d "/Library/LaunchDaemons" ]; then
+  source ./installer/install-for-launchd.sh
+  installable="true"
+fi
+if [ -z "$installable" ]; then
+  echo ""
+  echo "Unknown system service init type. You must install as a system service manually."
+  echo '(please file a bug with the output of "uname -a")'
+  echo ""
+fi

installer/install.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+set -e
+set -u
+
+### IMPORTANT ###
+###  VERSION  ###
+my_name=walnut
+my_app_pkg_name=org.oauth3.walnut.web
+my_app_ver="v1.2"
+my_azp_oauth3_ver="v1.2"
+# is the old version still needed in launchpad?
+#my_azp_oauth3_ver="v1.1.3"
+export NODE_VERSION="v8.9.0"
+
+if [ -z "${my_tmp-}" ]; then
+  my_tmp="$(mktemp -d)"
+  mkdir -p $my_tmp/opt/$my_name/core
+  echo "Installing to $my_tmp (will be moved after install)"
+  git clone ./ $my_tmp/opt/$my_name/core
+  pushd $my_tmp/opt/$my_name/core
+fi
+
+#################
+
+### IMPORTANT ###
+###  VERSION  ###
+#my_app_ver="v1.1"
+my_app_ver="v1.2"
+my_launchpad_ver="v1.2"
+my_iss_oauth3_rest_ver="v1.2.0"
+my_iss_oauth3_pages_ver="v1.2.1"
+my_www_ppl_ver=v1.0.15
+export NODE_VERSION="v8.9.0"
+#################
+export NODE_PATH=$my_tmp/opt/$my_name/lib/node_modules
+export PATH=$my_tmp/opt/$my_name/bin/:$PATH
+export NPM_CONFIG_PREFIX=$my_tmp/opt/$my_name
+my_npm="$NPM_CONFIG_PREFIX/bin/npm"
+#################
+
+
+
+# TODO un-hardcode core at al
+#my_app_dist=$my_tmp/opt/$my_name/lib/node_modules/$my_name/dist
+my_app_dist=$my_tmp/opt/$my_name/core/dist
+installer_base="https://git.coolaj86.com/coolaj86/goldilocks.js/raw/$my_app_ver"
+
+# Backwards compat
+# some scripts still use the old names
+my_app_dir=$my_tmp
+my_app_name=$my_name
+
+
+
+git checkout $my_app_ver
+
+mkdir -p $my_tmp/{etc,opt,srv,var}/$my_name
+mkdir -p "$my_tmp/var/log/$my_name"
+mkdir -p "$my_tmp/opt/$my_name"/{bin,config,core,etc,lib,node_modules,var}
+ln -s ../core/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name
+ln -s ../core/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name.js
+#ln -s ../lib/node_modules/$my_name/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name
+#ln -s ../lib/node_modules/$my_name/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name.js
+mkdir -p "$my_tmp/opt/$my_name"/packages/{api,pages,rest,services}
+mkdir -p "$my_tmp/opt/$my_name"/etc/client-api-grants
+# TODO move packages and sites to /srv, grants to /etc
+ln -s ../etc/client-api-grants "$my_tmp/opt/$my_name"/packages/client-api-grants
+mkdir -p "$my_tmp/opt/$my_name"/var/sites
+ln -s ../var/sites "$my_tmp/opt/$my_name"/packages/sites
+mkdir -p "$my_tmp/etc/$my_name"
+chmod 775 "$my_tmp/etc/$my_name"
+cat "$my_app_dist/etc/$my_name/$my_name.example.yml" > "$my_tmp/etc/$my_name/$my_name.example.yml"
+chmod 664 "$my_tmp/etc/$my_name/$my_name.example.yml"
+mkdir -p $my_tmp/var/log/$my_name
+
+
+
+#
+# Helpers
+#
+source ./installer/sudo-cmd.sh
+source ./installer/http-get.sh
+
+
+
+#
+# Dependencies
+#
+echo $NODE_VERSION > /tmp/NODEJS_VER
+# This will read the NODE_* and PATH variables set previously, as well as /tmp/NODEJS_VER
+http_bash "https://git.coolaj86.com/coolaj86/node-installer.sh/raw/v1.1/install.sh"
+$my_npm install -g npm@4
+$my_npm install -g bower
+touch $my_tmp/opt/$my_name/.bowerrc
+echo '{ "allow_root": true }' > $my_tmp/opt/$my_name/.bowerrc
+
+#pushd $my_tmp/opt/$my_name/lib/node_modules/$my_name
+pushd $my_tmp/opt/$my_name/core
+  mkdir -p ../node_modules
+  ln -s ../node_modules node_modules
+  $my_npm install
+popd
+
+git clone https://git.coolaj86.com/coolaj86/walnut_launchpad.html.git $my_tmp/opt/$my_name/core/lib/walnut@oauth3.org/setup
+pushd $my_tmp/opt/$my_name/core/lib/walnut@oauth3.org/setup
+  git pull
+  git checkout $my_launchpad_ver
+
+  git clone https://git.oauth3.org/OAuth3/oauth3.js.git ./assets/oauth3.org
+  pushd assets/oauth3.org
+    git checkout $my_azp_oauth3_ver
+  popd
+popd
+
+pushd $my_tmp/opt/$my_name/packages
+  git clone https://git.oauth3.org/OAuth3/issuer.rest.walnut.js.git rest/issuer@oauth3.org
+  pushd rest/issuer@oauth3.org/
+      git checkout $my_iss_oauth3_rest_ver
+      $my_npm install
+  popd
+
+  git clone https://git.oauth3.org/OAuth3/issuer.html.git pages/issuer@oauth3.org
+  pushd pages/issuer@oauth3.org
+    git checkout $my_iss_oauth3_pages_ver
+    bash ./install.sh
+
+    pushd ./assets/oauth3.org
+      git checkout $my_azp_oauth3_ver
+    popd
+  popd
+
+  git clone https://git.coolaj86.com/coolaj86/walnut_rest_www_oauth3.org.js.git rest/www@oauth3.org
+  pushd rest/www@oauth3.org
+    git checkout $my_www_ppl_ver
+    $my_npm install
+  popd
+popd
+
+
+
+#
+# System Service
+#
+source ./installer/my-root.sh
+echo "Pre-installation to $my_tmp complete, now installing to $my_root/ ..."
+set +e
+if type -p tree >/dev/null 2>/dev/null; then
+  #tree -I "node_modules|include|share" $my_tmp
+  tree -L 6 -I "include|share|npm" $my_tmp
+else
+  ls $my_tmp
+fi
+set -e
+
+source ./installer/my-user-my-group.sh
+echo "User $my_user Group $my_group"
+
+$sudo_cmd chown -R $my_user:$my_group $my_tmp
+$sudo_cmd chown root:root $my_tmp/*
+$sudo_cmd chown root:root $my_tmp
+$sudo_cmd chmod 0755 $my_tmp
+$sudo_cmd rsync -a --ignore-existing $my_tmp/ $my_root/
+$sudo_cmd rsync -a --ignore-existing $my_app_dist/etc/$my_name/$my_name.yml $my_root/etc/$my_name/$my_name.yml
+source ./installer/install-system-service.sh
+
+# Change to admin perms
+$sudo_cmd chown -R $my_user:$my_group $my_root/opt/$my_name
+$sudo_cmd chown -R $my_user:$my_group $my_root/var/www $my_root/srv/www
+
+# make sure the files are all read/write for the owner and group, and then set
+# the setuid and setgid bits so that any files/directories created inside these
+# directories have the same owner and group.
+$sudo_cmd chmod -R ug+rwX $my_root/opt/$my_name
+find $my_root/opt/$my_name -type d -exec $sudo_cmd chmod ug+s {} \;
+
+
+
+echo ""
+echo "You must have some set of domain set up to properly use goldilocks+walnut:"
+echo ""
+echo "  example.com"
+echo "  www.example.com"
+echo "  api.example.com"
+echo "  assets.example.com"
+echo "  cloud.example.com"
+echo "  api.cloud.example.com"
+echo ""
+echo "Check the WALNUT README.md for more info and how to set up /etc/goldilocks/goldilocks.yml"
+echo ""
+echo "Unistall: rm -rf /srv/walnut/ /var/walnut/ /etc/walnut/ /opt/walnut/ /var/log/walnut/ /etc/systemd/system/walnut.service /etc/tmpfiles.d/walnut.conf"
+
+
+
+rm -rf $my_tmp

installer/my-root.sh

@@ -0,0 +1,8 @@
+# something or other about android and tmux using PREFIX
+#: "${PREFIX:=''}"
+my_root=""
+if [ -z "${PREFIX-}" ]; then
+  my_root=""
+else
+  my_root="$PREFIX"
+fi

installer/my-user-my-group.sh

@@ -0,0 +1,19 @@
+if type -p adduser >/dev/null 2>/dev/null; then
+  if [ -z "$(cat $my_root/etc/passwd | grep $my_app_name)" ]; then
+    $sudo_cmd adduser --home $my_root/opt/$my_app_name --gecos '' --disabled-password $my_app_name
+  fi
+  my_user=$my_app_name
+  my_group=$my_app_name
+elif [ -n "$(cat /etc/passwd | grep www-data:)" ]; then
+  # Linux (Ubuntu)
+  my_user=www-data
+  my_group=www-data
+elif [ -n "$(cat /etc/passwd | grep _www:)" ]; then
+  # Mac
+  my_user=_www
+  my_group=_www
+else
+  # Unsure
+  my_user=$(whoami)
+  my_group=$(id -g -n)
+fi

installer/sudo-cmd.sh

@@ -0,0 +1,7 @@
+# Not every platform has or needs sudo, gotta save them O(1)s...
+sudo_cmd=""
+set +e
+if type -p sudo >/dev/null 2>/dev/null; then
+  ((EUID)) && [[ -z "${ANDROID_ROOT-}" ]] && sudo_cmd="sudo"
+fi
+set -e

lib/bootstrap.js

@@ -162,15 +162,17 @@ module.exports.create = function (app, xconfx, models) {
     // TODO How can we help apps handle this? token?
     // TODO allow apps to configure trustedDomains, auth, etc
     app.use('/api', cors);
-    app.get('/api/com.daplie.walnut.init', getConfig);
+    app.get('/api/walnut@daplie.com/init', getConfig);
-    app.post('/api/com.daplie.walnut.init', setConfig);
+    app.get('/api/com.daplie.walnut.init', getConfig); // deprecated
+    app.post('/api/walnut@daplie.com/init', setConfig);
+    app.post('/api/com.daplie.walnut.init', setConfig); // deprecated
 
     // TODO use package loader
-    //app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'com.daplie.walnut.init')));
+    //app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'walnut@daplie.com', 'init')));
-    app.use('/', express.static(path.join(__dirname, 'com.daplie.walnut.init')));
+    app.use('/', express.static(path.join(__dirname, 'walnut@daplie.com', 'init')));
     app.use('/', function (req, res, next) {
       res.statusCode = 404;
-      res.end('Walnut Bootstrap Not Found. Mising com.daplie.walnut.init');
+      res.end('Walnut Bootstrap Not Found. Mising walnut@daplie.com/init');
     });
 
     return new PromiseA(function (_resolve) {

lib/com.daplie.walnut.current/index.html

@@ -0,0 +1,8 @@
+<body style="background-color: black; color: white; font-family: Lato, Helvetica, 'Sans Serif';">
+  <br/>
+  <br/>
+  <br/>
+  <br/>
+  <br/>
+  <center><h1>Welcome!</h1></center>
+</body>

lib/com.daplie.walnut.current/scripts/daplie-pbkdf2.js

@@ -92,11 +92,11 @@
 
     // First, create a PBKDF2 "key" containing the passphrase
     return crypto.subtle.importKey(
-      "raw",
+      "raw"
-      Unibabel.utf8ToBuffer(nodeObj.secret),
+    , Unibabel.utf8ToBuffer(nodeObj.secret)
-      { "name": kdf.kdf },
+    , { "name": kdf.kdf }
-      false,
+    , false
-      ["deriveKey"]).
+    , ["deriveKey"]).
     // Derive a key from the password
     then(function (passphraseKey) {
       var keyconf = {

lib/com.daplie.walnut.current/scripts/pbkdf2.webcrypto.test.js

@@ -26,11 +26,11 @@
 
     // First, create a PBKDF2 "key" containing the password
     return crypto.subtle.importKey(
-      "raw",
+      "raw"
-      Unibabel.utf8ToBuffer(passphrase),
+    , Unibabel.utf8ToBuffer(passphrase)
-      { "name": kdfname },
+    , { "name": kdfname }
-      false,
+    , false
-      ["deriveKey"]).
+    , ["deriveKey"]).
     // Derive a key from the password
     then(function (passphraseKey) {
       return crypto.subtle.deriveKey(

lib/common.js

@@ -0,0 +1,33 @@
+'use strict';
+
+function rejectableRequest(req, res, promise, msg) {
+  return promise.error(function (err) {
+    res.error(err);
+  }).catch(function (err) {
+    console.error('[ERROR] \'' + msg + '\'');
+    // The stack contains the message as well, so no need to log the message when we log the stack
+    console.error(err.stack || err.message || JSON.stringify(err));
+
+    res.error(err);
+  });
+}
+module.exports.rejectableRequest = rejectableRequest;
+
+module.exports.promisableRequest =
+module.exports.promiseRequest = function promiseRequest(req, res, promise, msg) {
+  promise = promise.then(function (result) {
+    if (result._cache) {
+      res.setHeader('Cache-Control', 'public, max-age=' + (result._cache / 1000));
+      res.setHeader('Expires', new Date(Date.now() + result._cache).toUTCString());
+    }
+    if (result._mime) {
+      res.setHeader('Content-Type', result._mime);
+    }
+    if (result._value) {
+      result = result._value;
+    }
+    res.send(result);
+  });
+
+  return rejectableRequest(req, res, promise, msg);
+};

lib/main.js

@@ -1,6 +1,6 @@
 'use strict';
 
-module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi) {
+module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi, errorIfAssets) {
   var PromiseA = require('bluebird');
   var path = require('path');
   var fs = PromiseA.promisifyAll(require('fs'));
@@ -58,8 +58,8 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi
     }
 
     if (!setupApp) {
-      //setupApp = express.static(path.join(xconfx.staticpath, 'com.daplie.walnut'));
+      //setupApp = express.static(path.join(xconfx.staticpath, 'walnut@daplie.com'));
-      setupApp = express.static(path.join(__dirname, 'com.daplie.walnut'));
+      setupApp = express.static(path.join(__dirname, 'walnut@daplie.com', 'setup'));
     }
     setupApp(req, res, function () {
       if ('/' === req.url) {
@@ -293,10 +293,27 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi
   // TODO handle assets.example.com/sub/assets/com.example.xyz/
 
   app.use('/api', require('connect-send-error').error());
+  app.use('/assets', require('connect-send-error').error());
   app.use('/', function (req, res, next) {
-    // If this doesn't look like an API we can move along
+    // If this doesn't look like an API or assets we can move along
-    if (!/\/api(\/|$)/.test(req.url)) {
+
-      // /^api\./.test(req.hostname) &&
+    /*
+    console.log('.');
+    console.log('[main.js] req.url, req.hostname');
+    console.log(req.url);
+    console.log(req.hostname);
+    console.log('.');
+    */
+
+    if (!/\/(api|assets)(\/|$)/.test(req.url)) {
+      //console.log('[main.js] api|assets');
+      next();
+      return;
+    }
+
+    // keep https://assets.example.com/assets but skip https://example.com/assets
+    if (/\/assets(\/|$)/.test(req.url) && !/(^|\.)(api|assets)(\.)/.test(req.hostname) && !/^[0-9\.]+$/.test(req.hostname)) {
+      //console.log('[main.js] skip');
       next();
       return;
     }
@@ -325,6 +342,7 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi
     return;
   });
   app.use('/', errorIfApi);
+  app.use('/', errorIfAssets);
   app.use('/', serveStatic);
   app.use('/', serveApps);
 

lib/oauth3.js

@@ -0,0 +1,306 @@
+'use strict';
+
+var PromiseA = require('bluebird');
+
+function generateRescope(req, Models, decoded, fullPpid, ppid) {
+  return function (/*sub*/) {
+    // TODO: this function is supposed to convert PPIDs of different parties to some account
+    // ID that allows application to keep track of permisions and what-not.
+    console.log('[rescope] Attempting ', fullPpid);
+    return Models.IssuerOauth3OrgGrants.find({ azpSub: fullPpid }).then(function (results) {
+      if (results[0]) {
+        console.log('[rescope] lukcy duck: got it on the 1st try');
+        return PromiseA.resolve(results);
+      }
+
+      // XXX BUG XXX
+      // should be able to distinguish between own ids and 3rd party via @whatever.com
+      return Models.IssuerOauth3OrgGrants.find({ azpSub: ppid });
+    }).then(function (results) {
+      var result = results[0];
+
+      if (!result || !result.sub || !decoded.iss) {
+        // XXX BUG XXX TODO swap this external ppid for an internal (and ask user to link with existing profile)
+        //req.oauth3.accountIdx = fullPpid;
+        throw new Error("internal / external ID swapping not yet implemented. TODO: "
+          + "No profile found with that credential. Would you like to create a new profile or link to an existing profile?");
+      }
+
+      // XXX BUG XXX need to pass own url in to use as issuer for own tokens
+      req.oauth3.accountIdx = result.sub + '@' + decoded.iss;
+
+      console.log('[rescope] result:');
+      console.log(results);
+      console.log(req.oauth3.accountIdx);
+
+      return PromiseA.resolve(req.oauth3.accountIdx);
+    });
+  };
+}
+
+function extractAccessToken(req) {
+  var token = null;
+  var parts;
+  var scheme;
+  var credentials;
+
+  if (req.headers && req.headers.authorization) {
+    // Works for all of Authorization: Bearer {{ token }}, Token {{ token }}, JWT {{ token }}
+    parts = req.headers.authorization.split(' ');
+
+    if (parts.length !== 2) {
+      return PromiseA.reject(new Error("malformed Authorization header"));
+    }
+
+    scheme = parts[0];
+    credentials = parts[1];
+
+    if (-1 !== ['token', 'bearer'].indexOf(scheme.toLowerCase())) {
+      token = credentials;
+    }
+  }
+
+  if (req.body && req.body.access_token) {
+    if (token) { PromiseA.reject(new Error("token exists in header and body")); }
+    token = req.body.access_token;
+  }
+
+  // TODO disallow query with req.method === 'GET'
+  // NOTE: the case of DDNS on routers requires a GET and access_token
+  // (cookies should be used for protected static assets)
+  if (req.query && req.query.access_token) {
+    if (token) { PromiseA.reject(new Error("token already exists in either header or body and also in query")); }
+    token = req.query.access_token;
+  }
+
+  /*
+  err = new Error(challenge());
+  err.code = 'E_BEARER_REALM';
+
+  if (!token) { return PromiseA.reject(err); }
+  */
+
+  return PromiseA.resolve(token);
+}
+
+function verifyToken(token) {
+  var jwt = require('jsonwebtoken');
+  var decoded;
+
+  if (!token) {
+    return PromiseA.reject({
+      message: 'no token provided'
+    , code: 'E_NO_TOKEN'
+    , url: 'https://oauth3.org/docs/errors#E_NO_TOKEN'
+    });
+  }
+
+  try {
+    decoded = jwt.decode(token, {complete: true});
+  } catch (e) {}
+  if (!decoded) {
+    return PromiseA.reject({
+      message: 'provided token not a JSON Web Token'
+    , code: 'E_NOT_JWT'
+    , url: 'https://oauth3.org/docs/errors#E_NOT_JWT'
+    });
+  }
+
+  var sub = decoded.payload.sub || decoded.payload.ppid || decoded.payload.appScopedId;
+  if (!sub) {
+    return PromiseA.reject({
+      message: 'token missing sub'
+    , code: 'E_MISSING_SUB'
+    , url: 'https://oauth3.org/docs/errors#E_MISSING_SUB'
+    });
+  }
+  var kid = decoded.header.kid || decoded.payload.kid;
+  if (!kid) {
+    return PromiseA.reject({
+      message: 'token missing kid'
+    , code: 'E_MISSING_KID'
+    , url: 'https://oauth3.org/docs/errors#E_MISSING_KID'
+    });
+  }
+  if (!decoded.payload.iss) {
+    return PromiseA.reject({
+      message: 'token missing iss'
+    , code: 'E_MISSING_ISS'
+    , url: 'https://oauth3.org/docs/errors#E_MISSING_ISS'
+    });
+  }
+
+  var OAUTH3 = require('oauth3.js');
+  OAUTH3._hooks = require('oauth3.js/oauth3.node.storage.js');
+  return OAUTH3.discover(decoded.payload.iss).then(function (directives) {
+    var args = (directives || {}).retrieve_jwk;
+    if (typeof args === 'string') {
+      args = { url: args, method: 'GET' };
+    }
+    if (typeof (args || {}).url !== 'string') {
+      return PromiseA.reject({
+        message: 'token issuer does not support retrieving JWKs'
+      , code: 'E_INVALID_ISS'
+      , url: 'https://oauth3.org/docs/errors#E_INVALID_ISS'
+      });
+    }
+
+    var params = {
+      sub: sub
+    , kid: kid
+    };
+    var url = args.url;
+    var body;
+    Object.keys(params).forEach(function (key) {
+      if (url.indexOf(':'+key) !== -1) {
+        url = url.replace(':'+key, params[key]);
+        delete params[key];
+      }
+    });
+    if (Object.keys(params).length > 0) {
+      if ('GET' === (args.method || 'GET').toUpperCase()) {
+        url += '?' + OAUTH3.query.stringify(params);
+      } else {
+        body = params;
+      }
+    }
+
+    return OAUTH3.request({
+      url: OAUTH3.url.resolve(directives.api, url)
+    , method: args.method
+    , data: body
+    }).catch(function (err) {
+      return PromiseA.reject({
+        message: 'failed to retrieve public key from token issuer'
+      , code: 'E_NO_PUB_KEY'
+      , url: 'https://oauth3.org/docs/errors#E_NO_PUB_KEY'
+      , subErr: err.toString()
+      });
+    });
+  }, function (err) {
+    return PromiseA.reject({
+      message: 'token issuer is not a valid OAuth3 provider'
+    , code: 'E_INVALID_ISS'
+    , url: 'https://oauth3.org/docs/errors#E_INVALID_ISS'
+    , subErr: err.toString()
+    });
+  }).then(function (res) {
+    if (res.data.error) {
+      return PromiseA.reject(res.data.error);
+    }
+    var opts = {};
+    if (Array.isArray(res.data.alg)) {
+      opts.algorithms = res.data.alg;
+    } else if (typeof res.data.alg === 'string') {
+      opts.algorithms = [res.data.alg];
+    }
+
+    try {
+      return jwt.verify(token, require('jwk-to-pem')(res.data), opts);
+    } catch (err) {
+      return PromiseA.reject({
+        message: 'token verification failed'
+      , code: 'E_INVALID_TOKEN'
+      , url: 'https://oauth3.org/docs/errors#E_INVALID_TOKEN'
+      , subErr: err.toString()
+      });
+    }
+  });
+}
+
+function deepFreeze(obj) {
+  Object.keys(obj).forEach(function (key) {
+    if (obj[key] && typeof obj[key] === 'object') {
+      deepFreeze(obj[key]);
+    }
+  });
+  Object.freeze(obj);
+}
+
+function cookieOauth3(Models, req, res, next) {
+  req.oauth3 = {};
+
+  var token = req.cookies.jwt;
+
+  req.oauth3.encodedToken = token;
+  req.oauth3.verifyAsync = function (jwt) {
+    return verifyToken(jwt || token);
+  };
+
+  return verifyToken(token).then(function  (decoded) {
+    req.oauth3.token = decoded;
+    if (!decoded) {
+      return null;
+    }
+
+    var ppid = decoded.sub || decoded.ppid || decoded.appScopedId;
+    req.oauth3.ppid = ppid;
+    req.oauth3.accountIdx = ppid+'@'+decoded.iss;
+
+    var hash = require('crypto').createHash('sha256').update(req.oauth3.accountIdx).digest('base64');
+    hash = hash.replace(/\+/g, '-').replace(/\//g, '_').replace(/\=+/g, '');
+    req.oauth3.accountHash = hash;
+
+    req.oauth3.rescope = generateRescope(req, Models, decoded, fullPpid, ppid);
+  }).then(function () {
+    deepFreeze(req.oauth3);
+    //Object.defineProperty(req, 'oauth3', {configurable: false, writable: false});
+    next();
+  }, function (err) {
+    if ('E_NO_TOKEN' === err.code) {
+      next();
+      return;
+    }
+    console.error('[walnut] cookie lib/oauth3 error:');
+    console.error(err);
+    res.send(err);
+  });
+}
+
+function attachOauth3(Models, req, res, next) {
+  req.oauth3 = {};
+
+  extractAccessToken(req).then(function (token) {
+    req.oauth3.encodedToken = token;
+    req.oauth3.verifyAsync = function (jwt) {
+      return verifyToken(jwt || token);
+    };
+
+    if (!token) {
+      return null;
+    }
+    return verifyToken(token);
+  }).then(function  (decoded) {
+    req.oauth3.token = decoded;
+    if (!decoded) {
+      return null;
+    }
+
+    var ppid = decoded.sub || decoded.ppid || decoded.appScopedId;
+    var fullPpid = ppid+'@'+decoded.iss;
+    req.oauth3.ppid = ppid;
+
+    // TODO we can anonymize the relationship between our user as the other service's user
+    // in our own database by hashing the remote service's ppid and using that as the lookup
+    var hash = require('crypto').createHash('sha256').update(fullPpid).digest('base64');
+    hash = hash.replace(/\+/g, '-').replace(/\//g, '_').replace(/\=+/g, '');
+    req.oauth3.accountHash = hash;
+
+    req.oauth3.rescope = generateRescope(req, Models, decoded, fullPpid, ppid);
+
+    console.log('############### assigned req.oauth3:');
+    console.log(req.oauth3);
+  }).then(function () {
+    //deepFreeze(req.oauth3);
+    //Object.defineProperty(req, 'oauth3', {configurable: false, writable: false});
+    next();
+  }, function (err) {
+    console.error('[walnut] JWT lib/oauth3 error:');
+    console.error(err);
+    res.send(err);
+  });
+}
+
+module.exports.attachOauth3 = attachOauth3;
+module.exports.cookieOauth3 = cookieOauth3;
+module.exports.verifyToken = verifyToken;

lib/package-server-apis.js

@@ -55,19 +55,7 @@ function getApi(conf, pkgConf, pkgDeps, packagedApi) {
       packagedApi._api = require('express-lazy')();
       packagedApi._api_app = myApp;
 
-      //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
+      packagedApi._api.use('/', require('./oauth3').attachOauth3);
-      pkgDeps.getOauth3Controllers =
-      packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
-      require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
-
-      // DEBUG
-      //
-      /*
-      packagedApi._api.use('/', function (req, res, next) {
-        console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
-        next();
-      });
-      //*/
 
       // TODO fix backwards compat
 

lib/walnut@daplie.com/init/js/index.js

@@ -22,7 +22,7 @@ $(function () {
 
     return $.http({
       method: 'GET'
-    , url: baseUrl + '/api/com.daplie.walnut.init'
+    , url: baseUrl + '/api/walnut@daplie.com/init'
     , headers: {
         "Accept" : "application/json; charset=utf-8"
       }
@@ -83,7 +83,7 @@ $(function () {
 
     $.http({
       method: 'POST'
-    , url: baseUrl + '/api/com.daplie.walnut.init'
+    , url: baseUrl + '/api/walnut@daplie.com/init'
     , headers: {
         "Accept" : "application/json; charset=utf-8"
       , "Content-Type": "application/json; charset=utf-8"