Ind.ie is now Small Technology Foundation.
Commit 6dcf22fb authored by Aral Balkan's avatar Aral Balkan

Join hyperswarm and replicate with native node

parent 6fb7fc9b
//
// Hypha client
//
// Initial key generation
const session25519 = require('session25519')
const generateEFFDicewarePassphrase = require('eff-diceware-passphrase')
......
//
// Hypha: A native client for testing replication of a single hypercore.
//
const hypercore = require('hypercore')
const ram = require('random-access-memory')
const hyperswarm = require('@hyperswarm/network')
const { pipeline } = require('stream')
const { discoveryKey } = require('hypercore/lib/crypto')
const swarm = hyperswarm()
//////////////////////////////////////////////////////////////////////////////////////////
//
// Manually entered for now.
//
const readKeyInHex = '0b77c7fc5a7dd4be30bf8e7c02c2c1a87798f0218697ce07b2bce93a5da82ba4'
//
//////////////////////////////////////////////////////////////////////////////////////////
const readKeyBuffer = Buffer.from(readKeyInHex, 'hex')
const discoveryKeyBuffer = discoveryKey(readKeyBuffer)
const discoveryKeyInHex = discoveryKeyBuffer.toString('hex')
// Create the local hypercore instance
const localCore = hypercore((filename) => ram(filename), readKeyBuffer, {
createIfMissing: false,
overwrite: false,
valueEncoding: 'json',
onwrite: (index, data, peer, next) => {
// console.log(`Feed: [onWrite] index = ${index}, peer = ${peer}, data:`)
// console.log(data)
next()
}
})
const localReadStream = localCore.createReadStream({live: true})
localReadStream.on('data', (data) => {
console.log('[Replicate]', data)
})
localCore.on('ready', () => {
console.log('Local core ready.')
// HACK: Just for now, make sure we only connect once
let connected = false
//
// Join the swarm
//
swarm.join(discoveryKeyBuffer, {
lookup: true, // find and connect to peers.
announce: true // optional: announce self as a connection target.
})
swarm.on('connection', (remoteNativeStream, details) => {
// HACK: only handle first connection
if (connected) return
connected = true
console.log(`Joined swarm for read key ${readKeyInHex}, discovery key ${discoveryKeyInHex}`)
// Replicate!
console.log('About to replicate!')
// Create the local replication stream.
const localReplicationStream = localCore.replicate({
// TODO: why is Jim’s shopping list example setting encrypt to false?
// The encryption of __what__ does this affect?
// (I haven’t even tested this yet with it set to true to limit the variables.)
encrypt: false,
live: true
})
pipeline(
remoteNativeStream,
localReplicationStream,
remoteNativeStream,
(error) => {
console.log(`Pipe closed for ${readKeyInHex}`, error && error.message)
}
)
})
})
......@@ -190,6 +190,35 @@
"to-fast-properties": "^2.0.0"
}
},
"@hyperswarm/dht": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/@hyperswarm/dht/-/dht-0.0.1.tgz",
"integrity": "sha512-ocwfEAXVuiuqaMPNUcBVCZn9pguRYBTjCYiI/fwbnEYInzkPGo/KG/aWBSkKqN9lpKAWXRFLYJrgo7VDnAMCpg==",
"requires": {
"dht-rpc": "^4.0.1",
"ipv4-peers": "^1.1.1",
"protocol-buffers-encodings": "^1.1.0",
"record-cache": "^1.1.0"
}
},
"@hyperswarm/discovery": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@hyperswarm/discovery/-/discovery-1.2.0.tgz",
"integrity": "sha512-PJfsF/lflqmiEJZTmD1TtxR5ZBKL/DYMhOOfEko23KUnLyFD/7IYUTQ/C51vZAKFY6eRTvaU56DBBLeodDJkeA==",
"requires": {
"@hyperswarm/dht": "0.0.1",
"multicast-dns": "^7.2.0"
}
},
"@hyperswarm/network": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/@hyperswarm/network/-/network-0.0.4.tgz",
"integrity": "sha512-7+ove2aC16d3fDEunNsBmuS8tx1SQwuqtQn6O9HhCuTSe54iVsI6CnwGiSnJAFFY6lhghxNUV/CiqQyGoMxg7g==",
"requires": {
"@hyperswarm/discovery": "^1.1.0",
"utp-native": "^2.0.1"
}
},
"JSONStream": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz",
......@@ -1204,6 +1233,21 @@
"minimist": "^1.1.1"
}
},
"dht-rpc": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/dht-rpc/-/dht-rpc-4.1.1.tgz",
"integrity": "sha512-aNZbkZIBWDaCATP5WDAiiEKRvQ4qgBEAatQWwK4jOG+DR8cpdKTAerhV2O2a/rwg/e6fI9T5NY47l7Cy1k7FUQ==",
"requires": {
"codecs": "^1.2.1",
"ipv4-peers": "^1.1.1",
"k-bucket": "^5.0.0",
"protocol-buffers-encodings": "^1.1.0",
"sodium-universal": "^2.0.0",
"stream-collector": "^1.0.1",
"time-ordered-set": "^1.0.1",
"xor-distance": "^1.0.0"
}
},
"diffie-hellman": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
......@@ -1214,6 +1258,15 @@
"randombytes": "^2.0.0"
}
},
"dns-packet": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-4.2.0.tgz",
"integrity": "sha512-bn1AKpfkFbm0MIioOMHZ5qJzl2uypdBwI4nYNsqvhjsegBhcKJUlCrMPWLx6JEezRjxZmxhtIz/FkBEur2l8Cw==",
"requires": {
"ip": "^1.1.5",
"safe-buffer": "^5.1.1"
}
},
"domain-browser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
......@@ -2488,6 +2541,11 @@
"ipaddr.js": "^1.5.2"
}
},
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"ip-regex": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
......@@ -2498,6 +2556,11 @@
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.1.tgz",
"integrity": "sha1-+kt5+kf9Pe9eOxWYJRYcClGclCc="
},
"ipv4-peers": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ipv4-peers/-/ipv4-peers-1.1.1.tgz",
"integrity": "sha1-hauGKg9SFFg+BRh2P34dd1riTXY="
},
"is-accessor-descriptor": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
......@@ -2710,6 +2773,14 @@
"resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
"integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA="
},
"k-bucket": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/k-bucket/-/k-bucket-5.0.0.tgz",
"integrity": "sha512-r/q+wV/Kde62/tk+rqyttEJn6h0jR7x+incdMVSYTqK73zVxVrzJa70kJL49cIKen8XjIgUZKSvk8ktnrQbK4w==",
"requires": {
"randombytes": "^2.0.3"
}
},
"kind-of": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
......@@ -3043,6 +3114,15 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"multicast-dns": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.0.tgz",
"integrity": "sha512-Tu2QORGOFANB124NWQ/JTRhMf/ODouVLEuvu5Dz8YWEU55zQgRgFGnBHfIh5PbfNDAuaRl7yLB+pgWhSqVxi2Q==",
"requires": {
"dns-packet": "^4.0.0",
"thunky": "^1.0.2"
}
},
"nan": {
"version": "2.12.1",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz",
......@@ -3072,6 +3152,11 @@
"to-regex": "^3.0.1"
}
},
"napi-macros": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-1.8.2.tgz",
"integrity": "sha512-Tr0DNY4RzTaBG2W2m3l7ZtFuJChTH6VZhXVhkGGjF/4cZTt+i8GcM9ozD+30Lmr4mDoZ5Xx34t2o4GJqYWDGcg=="
},
"negotiator": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
......@@ -3085,8 +3170,7 @@
"node-gyp-build": {
"version": "3.7.0",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.7.0.tgz",
"integrity": "sha512-L/Eg02Epx6Si2NXmedx+Okg+4UHqmaf3TNcxd50SF9NQGcJaON3AtU++kax69XV7YWz4tUspqZSAsVofhFKG2w==",
"optional": true
"integrity": "sha512-L/Eg02Epx6Si2NXmedx+Okg+4UHqmaf3TNcxd50SF9NQGcJaON3AtU++kax69XV7YWz4tUspqZSAsVofhFKG2w=="
},
"normalize-path": {
"version": "2.1.1",
......@@ -3633,6 +3717,11 @@
}
}
},
"record-cache": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/record-cache/-/record-cache-1.1.0.tgz",
"integrity": "sha512-u8rbtLEJV7HRacl/ZYwSBFD8NFyB3PfTTfGLP37IW3hftQCwu6z4Q2RLyxo1YJUNRTEzJfpLpGwVuEYdaIkG9Q=="
},
"regex-cache": {
"version": "0.4.4",
"resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.4.tgz",
......@@ -4176,6 +4265,14 @@
"readable-stream": "^2.0.2"
}
},
"stream-collector": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-collector/-/stream-collector-1.0.1.tgz",
"integrity": "sha1-TU5V8XE1YSGyxfZVn5RHBaso2xU=",
"requires": {
"once": "^1.3.1"
}
},
"stream-combiner2": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz",
......@@ -4305,6 +4402,16 @@
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.3.tgz",
"integrity": "sha512-YwT8pjmNcAXBZqrubu22P4FYsh2D4dxRmnWBOL8Jk8bUcRUtc5326kx32tuTmFDAZtLOGEVNl8POAR8j896Iow=="
},
"time-ordered-set": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/time-ordered-set/-/time-ordered-set-1.0.2.tgz",
"integrity": "sha512-vGO99JkxvgX+u+LtOKQEpYf31Kj3i/GNwVstfnh4dyINakMgeZCpew1e3Aj+06hEslhtHEd52g7m5IV+o1K8Mw=="
},
"timeout-refresh": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/timeout-refresh/-/timeout-refresh-1.0.0.tgz",
"integrity": "sha512-y5ajDPPtyhumr7xRnQgOMMVR5/EXMCVHPwM7RWnMUZx9UzT8FGRBtwG4/rh2AWHkDr7JR8dBHU6NDGx7tEiEAg=="
},
"timers-browserify": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz",
......@@ -4556,6 +4663,30 @@
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"utp-native": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/utp-native/-/utp-native-2.1.3.tgz",
"integrity": "sha512-lXjmAJxlaz58GDFlEqKYkNzO5rqttA+/TVHl7UUAs8Saj1QJq/3D4IckuVpsmCsjjZod5N7sE8QMUCYScVHDpg==",
"requires": {
"napi-macros": "^1.8.1",
"node-gyp-build": "^3.5.0",
"readable-stream": "^3.0.2",
"timeout-refresh": "^1.0.0",
"unordered-set": "^2.0.1"
},
"dependencies": {
"readable-stream": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz",
"integrity": "sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"varint": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/varint/-/varint-4.0.1.tgz",
......@@ -4698,6 +4829,11 @@
"ultron": "1.0.x"
}
},
"xor-distance": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/xor-distance/-/xor-distance-1.0.0.tgz",
"integrity": "sha1-2nNdmyT8yo282bN00W0qAe6VQcY="
},
"xsalsa20": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/xsalsa20/-/xsalsa20-1.0.2.tgz",
......
......@@ -20,6 +20,7 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@babel/core": "^7.2.2",
"@hyperswarm/network": "0.0.4",
"babelify": "^10.0.0",
"budo": "github:aral/budo",
"buffer": "^5.2.1",
......
//
// Hypha server.
//
const fs = require('fs')
const https = require('https')
const { pipeline } = require('stream')
......@@ -7,6 +10,7 @@ const expressWebSocket = require('express-ws')
const websocketStream = require('websocket-stream/stream')
const ram = require('random-access-memory')
const hypercore = require('hypercore')
const hyperswarm = require('@hyperswarm/network')
const budo = require('budo')
const babelify = require('babelify')
......@@ -41,7 +45,9 @@ server.on('connect', (event) => {
// Add web socket routes.
router.ws('/hypha/:readKey', (webSocket, request) => {
const readKey = request.params.readKey
console.log('Got web socket request for ', readKey)
if (hypercores[readKey] !== undefined) {
......@@ -63,10 +69,10 @@ server.on('connect', (event) => {
newCore.on('ready', () => {
console.log(`Hypercore ready (${readKey})`)
const remoteStream = websocketStream(webSocket)
const localReadStream = newCore.createReadStream({live: true})
const remoteWebStream = websocketStream(webSocket)
const localReadStream = newCore.createReadStream({live: true})
localReadStream.on('data', (data) => {
console.log('[Replicate]', data)
})
......@@ -80,13 +86,50 @@ server.on('connect', (event) => {
})
pipeline(
remoteStream,
remoteWebStream,
localReplicationStream,
remoteStream,
remoteWebStream,
(error) => {
console.log(`Pipe closed for ${readKey}`, error && error.message)
}
)
//
// Connect to the hyperswarm for this hypercore.
//
const swarm = hyperswarm()
const discoveryKey = newCore.discoveryKey
const discoveryKeyInHex = discoveryKey.toString('hex')
console.log(discoveryKeyInHex)
console.log('About to join the swarm!')
// Join the swarm
swarm.join(newCore.discoveryKey, {
lookup: true, // find and connect to peers.
announce: true // optional: announce self as a connection target.
})
swarm.on('connection', (remoteNativeStream, details) => {
console.log(`Joined swarm for read key ${readKey} (discovery key: ${discoveryKeyInHex})`)
// Create a new replication stream
const nativeReplicationStream = newCore.replicate({
encrypt: false,
live: true
})
// Replicate!
pipeline(
remoteNativeStream,
nativeReplicationStream,
remoteNativeStream,
(error) => {
console.log(`(Native stream from swarm) Pipe closed for ${readKey}`, error && error.message)
}
)
})
})
})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment