Ind.ie is now Small Technology Foundation.
Commit d9924709 authored by Aral Balkan's avatar Aral Balkan

Integrated migrations into the startup sequence. Updated interface of database...

Integrated migrations into the startup sequence. Updated interface of database object — we now have access to various ‘flavours’ of the database to make it easier to use — with and without sublevels and with and without promises. Timeline order is now correct for all timelines.
parent e720341f
......@@ -44,6 +44,8 @@ StreamWeaver = require './StreamWeaver'
MainModel = require './MainModel'
Migrations = require './Migrations'
# Implement properties (with getter/setters) on the Function prototype
# (Thanks https://gist.github.com/reversepanda/5814547)
......@@ -55,6 +57,9 @@ class Main
instance = null
# Data schema version
dataSchemaVersion: 1
nodeSocket: null
pulseProcess: null
pulseAPIKey: null
......@@ -124,23 +129,23 @@ class Main
# Run database migrations.
#
@runDatabaseMigrations()
#
# Start the services.
#
(new Migrations @config).runMigrations().then =>
if @startAllServices
#
# Start all services.
#
@startAll()
else
#
# Only start NodeSocket.
# Start the services.
#
console.log 'ℹ️ In setup process; only starting the NodeSocket service.'
@startNodeSocket()
if @startAllServices
#
# Start all services.
#
@startAll()
else
#
# Only start NodeSocket.
#
console.log 'ℹ️ In setup process; only starting the NodeSocket service.'
@startNodeSocket()
# Save reference to the singleton instance.
......@@ -148,14 +153,6 @@ class Main
return instance
#
# Run database migrations
#
runDatabaseMigrations: =>
currentDatabaseVersion = 1
#
# Parses the command-line arguments and creates the main configuration object.
......@@ -253,16 +250,22 @@ class Main
aboutFolder = path.join publicFolder, "about"
databaseFolder = path.join homeDirectory, 'db'
@config =
homeDirectory: homeDirectory,
publicFolder: publicFolder,
privateFolder: privateFolder,
allFriendsFolder: allFriendsFolder,
allFriendsToFolder: allFriendsToFolder,
allFriendsFromFolder: allFriendsFromFolder,
specificFriendsFolder: specificFriendsFolder,
specificFriendsFolder: specificFriendsFolder,
homeDirectory: homeDirectory
publicFolder: publicFolder
privateFolder: privateFolder
allFriendsFolder: allFriendsFolder
allFriendsToFolder: allFriendsToFolder
allFriendsFromFolder: allFriendsFromFolder
specificFriendsFolder: specificFriendsFolder
specificFriendsFolder: specificFriendsFolder
aboutFolder: aboutFolder
syncFolder: syncFolder
databaseFolder: databaseFolder
dataSchemaVersion: @dataSchemaVersion
# Set the home directory on Pulse Config
PulseConfig.setHomeDirectory homeDirectory
......
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
Promise = require 'thrush'
WriteStream = require 'write-stream'
path = require 'path-extra'
fs = Promise.promisifyAll(require 'fs-extra')
targz = require 'tar.gz'
walk = require './walk'
class Migrations
instance = null
db: require './database'
homeFolder: null
syncFolder: null
databaseFolder: null
dataSchemaVersion: null
meta: null
dbFolderSnapshotPath: null
syncFolderSnapshotPath: null
dataSchemaVersionInDatabase: 0
constructor: (config) ->
# Singleton access
if instance
return instance
#
# Initialise the singleton instance
#
console.log 'Initialising Migrations singleton.'
@homeFolder = config.homeDirectory
@syncFolder = config.syncFolder
@databaseFolder = config.databaseFolder
@dataSchemaVersion = config.dataSchemaVersion
console.log "CURRENT Data schema version: #{@dataSchemaVersion}"
return instance
runMigrations: () =>
# Using LevelUp directly as level-sublevel can’t create a read stream on the root
# that iterates into the child sublevels.
@meta = @db.withSublevels.sublevel 'meta'
@meta.get 'dataSchemaVersion'
.then (dataSchemaVersionInDatabase) =>
console.log "Data schema version in database: #{dataSchemaVersionInDatabase}"
# So much for raising an error if a key doesn’t exist :)
if dataSchemaVersionInDatabase == ''
@dataSchemaVersionInDatabase = 0
else
@dataSchemaVersionInDatabase = dataSchemaVersionInDatabase
.catch (error) =>
# Not really an error; person just has earlier version
# of Heartbeat than the first one where we implemented migrations.
console.log "No data schema version in database yet. (Version 0)."
.finally =>
if "#{@dataSchemaVersionInDatabase}" == "#{@dataSchemaVersion}"
console.log "Heartbeat data schema version is up to date (#{@dataSchemaVersionInDatabase}). Not running migrations."
return
else
#
# Backup the current configuration so we can revert to it if something goes wrong.
#
@dbFolderSnapshotPath = path.join @homeFolder, "db-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"
@syncFolderSnapshotPath = path.join @homeFolder, "Sync-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"
targz().compress @databaseFolder, @dbFolderSnapshotPath
.then =>
targz().compress @syncFolder, @syncFolderSnapshotPath
.then =>
console.log "Current configuration snapshot complete."
#
# Run migrations
#
console.log ">>> #{@dataSchemaVersionInDatabase}"
console.log "Running migrations: #{[@dataSchemaVersionInDatabase...@dataSchemaVersion]}"
Promise.series [@dataSchemaVersionInDatabase...@dataSchemaVersion], (migrationVersion) =>
# Note migration versions are always numbered in sequence (0,1,2,3…)
# If the app is on data schema version 0 and the current version is 3, it will go through 0->1, 1->2, 2->3.
console.log "Running migration from version #{migrationVersion} to #{migrationVersion+1}"
console.log "Selector: migrateVersion#{migrationVersion}"
@["migrateVersion#{migrationVersion}"]()
.then =>
@meta.put 'dataSchemaVersion', @dataSchemaVersion
.then =>
console.log "Migrations complete. Updated app’s data schema version to #{@dataSchemaVersion}."
.catch (error) =>
# Migrations failed.
# Restore the initial configuration from our snapshot.
console.log "Migrations failed. Restoring initial configuration from snapshot. #{error}"
targz().extract @dbFolderSnapshotPath, @databaseFolder
.then =>
targz().extract @syncFolderSnapshotPath, @syncFolder
.then =>
console.log "Snapshot restored for version #{@dataSchemaVersionInDatabase}."
# Bail — we cannot migrate the data schema to the latest version. We can’t recover from this.
# Person either has to use the older version or file a bug report.
throw new Error "Migrations failed. Cannot continue."
.catch (error) =>
# Backing up the sync folders failed. Bail.
# Not going to risk messing up the person’s current configuration / data.
throw new Error "Sync folder migration snapshot failed. Cannot continue."
.catch (error) =>
# Backing up the database folders failed. Bail.
# Not going to risk messing up the person’s current configuration / data.
throw new Error "Database migration snapshot failed. Cannot continue."
#
# Version 0 to version 1.
#
# Removes the local message clocks from database keys and from message folder names.
#
depracatedMessageClockMatcher: /\d{9}-/g
migrateVersion0: =>
console.log "Running data schema migration from version 0 to version 1."
@removeMessageClockFromDatabase()
.then @removeMessageClockFromMessageFolderNames
.then @removeMessageClockFromMessageFiles
.then =>
console.log '✅ Version 0 to 1: data schema migration complete.'
removeMessageClockFromMessageFolderNames: =>
console.log 'Promising to remove message clocks from message folder names.'
numberOfFoldersAffected = 0
walk.folders @syncFolder
.then (folders) =>
Promise.series folders, (folder) =>
matches = folder.match @depracatedMessageClockMatcher
if matches != null
newFolder = folder.replace @depracatedMessageClockMatcher, ''
fs.moveAsync folder, newFolder
.then =>
numberOfFoldersAffected++
.then =>
console.log "✅ Updated #{numberOfFoldersAffected} message folder names out of #{folders.length} folders."
removeMessageClockFromMessageFiles: =>
console.log 'Promising to remove message clocks from message files.'
numberOfMessageFilesAffected = 0
walk.files @syncFolder
.then (files) =>
Promise.series files, (file) =>
if file.substr(-10) == 'index.html'
fs.readFileAsync file, {encoding: 'utf8'}
.then (message) =>
matches = message.match @depracatedMessageClockMatcher
if matches != null
message = message.replace @depracatedMessageClockMatcher, ''
fs.writeFileAsync file, message
.then =>
# console.log "Removed message clock from message in #{file}."
numberOfMessageFilesAffected++
.then =>
console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."
removeMessageClockFromDatabase: =>
console.log 'Promising to remove message clocks from database.'
numberOfDatabaseKeysAffected = 0
return new Promise ((fulfill, reject) =>
# Create options for a new stream
options = { gt: '\x00', lt: '\uffff' }
dataStream = @db.plain.createReadStream options
dataStream.on 'error', (err) =>
console.log "Data stream error: #{err}"
toArray = WriteStream.toArray (data) =>
Promise.series data, (datum) =>
matches = datum.key.match @depracatedMessageClockMatcher
if matches != null
# Remove the message clock from message keys.
newKey = datum.key.replace @depracatedMessageClockMatcher, ''
# Remove the message clock from message values.
newValue = datum.value.replace @depracatedMessageClockMatcher, ''
# console.log "Migrating #{datum.key} to #{newKey}…"
# console.log "Value:"
# console.log datum.value
operations = [
{type: 'put', key: newKey, value: newValue},
{type: 'del', key: datum.key}
]
# Carry out the operations in a batch.
@db.plain.batch(operations)
.then =>
# console.log "OK."
numberOfDatabaseKeysAffected++
.catch (error) =>
console.log "Error while updating #{datum.key} to #{newKey}: #{error}"
.then =>
console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."
process.nextTick(=>
console.log "Database migration complete."
fulfill true
)
.catch (error) =>
console.log "Could not migrate the database: #{error}."
dataStream.pipe toArray
)
module.exports = Migrations
\ No newline at end of file
......@@ -28,7 +28,7 @@ WriteStream = require 'write-stream'
liveStream = require 'level-live-stream'
database = require './database'
database = (require './database').withSublevels
class StreamWeaver
......@@ -209,7 +209,7 @@ class StreamWeaver
#
# Also known as document/blog order. All other
# timelines are displayed in conversation order (latest post last).
options.reverse = !(timeline == @timeline.public || timeline == @timeline.private || timeline == @timeline.allFriends)
options.reverse = (timeline == @timeline.public || timeline == @timeline.private || timeline == @timeline.allFriends)
# console.log "TIMELINE REVERSED? #{options.reverse}"
# console.log "@timelines[timeline] = #{@timelines[timeline]}"
......
################################################################################
##########################################################################################
#
# Ind.ie Heartbeat Node
#
# Database — singleton access to the LevelDB database.
#
# This is a singleton object that proxies a reference to the database instance.
# This is a singleton object that proxies references to the LevelDB database.
#
# Usage:
#
# database = require './database'
#
# database.plain # Reference to plain LevelUp database
# database.withSublevels # level-sublevel-wrapped reference
# database.withPromises # LevelUp promisified with Thrush
# database.withSublevelsAndPromises # Wrapped in level-sublevel & promisified with Thrush
#
# This is independent technology. See ind.ie/manifesto
#
......@@ -19,16 +28,20 @@
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
##########################################################################################
path = require 'path-extra'
LevelUp = require 'levelup'
Sublevel = require 'level-sublevel'
LevelUpThrushPromisify = require './levelup-thrush-promisify'
class Database
instance = null
db: null
plain: null
withSublevels: null
withPromises: null
withSublevelsAndPromises: null
constructor: ->
......@@ -46,7 +59,11 @@ class Database
databasePath = 'db'
console.log "Waystone LevelDB database path: #{databasePath}"
@db = Sublevel (LevelUp databasePath)
@plain = LevelUp databasePath
@withSublevels = Sublevel @plain
@withPromises = LevelUpThrushPromisify @plain
@withSublevelsAndPromises = LevelUpThrushPromisify @withSublevels
# Save reference to the singleton instance.
instance = @
......@@ -56,4 +73,4 @@ class Database
#
# Actually proxy the database instance to cut down on boilerploit code when used.
#
module.exports = (new Database).db
module.exports = new Database
/*
Promisifies LevelUp
Pass either a LevelUp instance or a level-sublevel instance of the database.
Based on Level Bluebird Promise by Alex Ferreira
(Original: https://github.com/alexferreira/level-bluebird-promise)
Copyright © Aral Balkan.
*/
var _ = require('lodash')
, Promise = require('thrush')
, Manifest = require('level-manifest')
module.exports = function (db) {
manifest = new Manifest(db);
return _resursive(db, manifest);
};
function _resursive(db, manifest){
_.chain(manifest.methods).pick(function(obj){
return obj.type == 'async';
}).each(function(v, k){
if (db[k] != undefined)
{
return db[k] = Promise.promisify(db[k]);
}
}).value();
var sublevels = manifest.sublevels || {};
for (var name in sublevels) if (_.has(sublevels, name)){
_resursive(db.sublevels[name], sublevels[name]);
}
if (_.isFunction(db.sublevel)) {
var Sub = db.sublevel;
db.sublevel = function(name) {
var sublevel = Sub.apply(this, arguments);
if (!_.has(sublevels, name)) _resursive(sublevel, new Manifest(sublevel));
return sublevel;
}
}
return db;
};
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
Promise = require 'thrush'
LevelUp = require 'levelup'
LevelPromisify = require 'level-promisify'
WriteStream = require 'write-stream'
fs = Promise.promisifyAll(require 'fs-extra')
walk = require './walk'
#level = LevelUp 'db'
# LOCAL TEST
level = LevelUp '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/db'
db = LevelPromisify level
#
# Version 0 to version 1.
#
# Removes the local message clocks from database keys and from message folder names.
#
depracatedMessageClockMatcher = /\d{9}-/g
version0toVersion1 = ->
removeMessageClockFromDatabase()
.then removeMessageClockFromMessageFolderNames
.then removeMessageClockFromMessageFiles
.then ->
console.log '✅ Version 0 to 1: data schema migration complete.'
removeMessageClockFromMessageFolderNames = ->
console.log 'Promising to remove message clocks from message folder names.'
numberOfFoldersAffected = 0
(walk.folders '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
matches = file.match depracatedMessageClockMatcher
if matches != null
newFile = file.replace depracatedMessageClockMatcher, ''
# console.log "From: #{file}\nTo: #{newFile}\n\n"
fs.moveAsync file, newFile
.then ->
# console.log "Moved #{file} to #{newFile}"
numberOfFoldersAffected++
.then ->
console.log "✅ Migrated #{numberOfFoldersAffected} folder names out of #{files.length} files."
removeMessageClockFromMessageFiles = ->
console.log 'Promising to remove message clocks from message files.'
numberOfMessageFilesAffected = 0
(walk.files '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
if file.substr(-10) == 'index.html'
fs.readFileAsync file, {encoding: 'utf8'}
.then (message) ->
console.log "Checking #{file}"
matches = message.match depracatedMessageClockMatcher
if matches != null
message = message.replace depracatedMessageClockMatcher, ''
fs.writeFileAsync file, message
.then ->
console.log "Removed message clock from message in #{file}."
numberOfMessageFilesAffected++
.then ->
console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."
removeMessageClockFromDatabase = ->
console.log 'Promising to remove message clocks from database.'
numberOfDatabaseKeysAffected = 0
return new Promise ((fulfill, reject) ->
# Create options for a new stream
options = { gt: '\x00', lt: '\uffff' }
dataStream = db.root.createReadStream options
dataStream.on 'error', (err) ->
console.log "Data stream error: #{err}"
toArray = WriteStream.toArray (data) ->
Promise.series data, (datum) ->
matches = datum.key.match depracatedMessageClockMatcher
if matches != null
# Remove the message clock from message keys.
newKey = datum.key.replace depracatedMessageClockMatcher, ''
# Remove the message clock from message values.
newValue = datum.value.replace depracatedMessageClockMatcher, ''
# console.log "Migrating #{datum.key} to #{newKey}…"
# console.log "Value:"
# console.log datum.value
operations = [
{type: 'put', key: newKey, value: newValue},
{type: 'del', key: datum.key}
]
# Carry out the operations in a batch.
db.batch(operations)
.then ->
# console.log "OK."
numberOfDatabaseKeysAffected++
.catch (error) ->
console.log "Error while updating #{datum.key} to #{newKey}: #{error}"
.then ->
console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."
process.nextTick(->
console.log "Database migration complete."
fulfill true
)
.catch (error) ->
console.log "Could not migrate the database: #{error}."
dataStream.pipe toArray
)
# TEST
version0toVersion1()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment