Commit d9924709 authored by Aral Balkan's avatar Aral Balkan

Integrated migrations into the startup sequence. Updated interface of database...

Integrated migrations into the startup sequence. Updated interface of database object — we now have access to various ‘flavours’ of the database to make it easier to use — with and without sublevels and with and without promises. Timeline order is now correct for all timelines.
parent e720341f
......@@ -44,6 +44,8 @@ StreamWeaver = require './StreamWeaver'
MainModel = require './MainModel'
Migrations = require './Migrations'
# Implement properties (with getter/setters) on the Function prototype
# (Thanks https://gist.github.com/reversepanda/5814547)
......@@ -55,6 +57,9 @@ class Main
instance = null
# Data schema version
dataSchemaVersion: 1
nodeSocket: null
pulseProcess: null
pulseAPIKey: null
......@@ -124,23 +129,23 @@ class Main
# Run database migrations.
#
@runDatabaseMigrations()
#
# Start the services.
#
(new Migrations @config).runMigrations().then =>
if @startAllServices
#
# Start all services.
#
@startAll()
else
#
# Only start NodeSocket.
# Start the services.
#
console.log 'ℹ️ In setup process; only starting the NodeSocket service.'
@startNodeSocket()
if @startAllServices
#
# Start all services.
#
@startAll()
else
#
# Only start NodeSocket.
#
console.log 'ℹ️ In setup process; only starting the NodeSocket service.'
@startNodeSocket()
# Save reference to the singleton instance.
......@@ -148,14 +153,6 @@ class Main
return instance
#
# Run database migrations
#
runDatabaseMigrations: =>
currentDatabaseVersion = 1
#
# Parses the command-line arguments and creates the main configuration object.
......@@ -253,16 +250,22 @@ class Main
aboutFolder = path.join publicFolder, "about"
databaseFolder = path.join homeDirectory, 'db'
@config =
homeDirectory: homeDirectory,
publicFolder: publicFolder,
privateFolder: privateFolder,
allFriendsFolder: allFriendsFolder,
allFriendsToFolder: allFriendsToFolder,
allFriendsFromFolder: allFriendsFromFolder,
specificFriendsFolder: specificFriendsFolder,
specificFriendsFolder: specificFriendsFolder,
homeDirectory: homeDirectory
publicFolder: publicFolder
privateFolder: privateFolder
allFriendsFolder: allFriendsFolder
allFriendsToFolder: allFriendsToFolder
allFriendsFromFolder: allFriendsFromFolder
specificFriendsFolder: specificFriendsFolder
specificFriendsFolder: specificFriendsFolder
aboutFolder: aboutFolder
syncFolder: syncFolder
databaseFolder: databaseFolder
dataSchemaVersion: @dataSchemaVersion
# Set the home directory on Pulse Config
PulseConfig.setHomeDirectory homeDirectory
......
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
Promise = require 'thrush'
WriteStream = require 'write-stream'
path = require 'path-extra'
fs = Promise.promisifyAll(require 'fs-extra')
targz = require 'tar.gz'
walk = require './walk'
class Migrations
instance = null
db: require './database'
homeFolder: null
syncFolder: null
databaseFolder: null
dataSchemaVersion: null
meta: null
dbFolderSnapshotPath: null
syncFolderSnapshotPath: null
dataSchemaVersionInDatabase: 0
constructor: (config) ->
# Singleton access
if instance
return instance
#
# Initialise the singleton instance
#
console.log 'Initialising Migrations singleton.'
@homeFolder = config.homeDirectory
@syncFolder = config.syncFolder
@databaseFolder = config.databaseFolder
@dataSchemaVersion = config.dataSchemaVersion
console.log "CURRENT Data schema version: #{@dataSchemaVersion}"
return instance
runMigrations: () =>
# Using LevelUp directly as level-sublevel can’t create a read stream on the root
# that iterates into the child sublevels.
@meta = @db.withSublevels.sublevel 'meta'
@meta.get 'dataSchemaVersion'
.then (dataSchemaVersionInDatabase) =>
console.log "Data schema version in database: #{dataSchemaVersionInDatabase}"
# So much for raising an error if a key doesn’t exist :)
if dataSchemaVersionInDatabase == ''
@dataSchemaVersionInDatabase = 0
else
@dataSchemaVersionInDatabase = dataSchemaVersionInDatabase
.catch (error) =>
# Not really an error; person just has earlier version
# of Heartbeat than the first one where we implemented migrations.
console.log "No data schema version in database yet. (Version 0)."
.finally =>
if "#{@dataSchemaVersionInDatabase}" == "#{@dataSchemaVersion}"
console.log "Heartbeat data schema version is up to date (#{@dataSchemaVersionInDatabase}). Not running migrations."
return
else
#
# Backup the current configuration so we can revert to it if something goes wrong.
#
@dbFolderSnapshotPath = path.join @homeFolder, "db-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"
@syncFolderSnapshotPath = path.join @homeFolder, "Sync-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"
targz().compress @databaseFolder, @dbFolderSnapshotPath
.then =>
targz().compress @syncFolder, @syncFolderSnapshotPath
.then =>
console.log "Current configuration snapshot complete."
#
# Run migrations
#
console.log ">>> #{@dataSchemaVersionInDatabase}"
console.log "Running migrations: #{[@dataSchemaVersionInDatabase...@dataSchemaVersion]}"
Promise.series [@dataSchemaVersionInDatabase...@dataSchemaVersion], (migrationVersion) =>
# Note migration versions are always numbered in sequence (0,1,2,3…)
# If the app is on data schema version 0 and the current version is 3, it will go through 0->1, 1->2, 2->3.
console.log "Running migration from version #{migrationVersion} to #{migrationVersion+1}"
console.log "Selector: migrateVersion#{migrationVersion}"
@["migrateVersion#{migrationVersion}"]()
.then =>
@meta.put 'dataSchemaVersion', @dataSchemaVersion
.then =>
console.log "Migrations complete. Updated app’s data schema version to #{@dataSchemaVersion}."
.catch (error) =>
# Migrations failed.
# Restore the initial configuration from our snapshot.
console.log "Migrations failed. Restoring initial configuration from snapshot. #{error}"
targz().extract @dbFolderSnapshotPath, @databaseFolder
.then =>
targz().extract @syncFolderSnapshotPath, @syncFolder
.then =>
console.log "Snapshot restored for version #{@dataSchemaVersionInDatabase}."
# Bail — we cannot migrate the data schema to the latest version. We can’t recover from this.
# Person either has to use the older version or file a bug report.
throw new Error "Migrations failed. Cannot continue."
.catch (error) =>
# Backing up the sync folders failed. Bail.
# Not going to risk messing up the person’s current configuration / data.
throw new Error "Sync folder migration snapshot failed. Cannot continue."
.catch (error) =>
# Backing up the database folders failed. Bail.
# Not going to risk messing up the person’s current configuration / data.
throw new Error "Database migration snapshot failed. Cannot continue."
#
# Version 0 to version 1.
#
# Removes the local message clocks from database keys and from message folder names.
#
depracatedMessageClockMatcher: /\d{9}-/g
migrateVersion0: =>
console.log "Running data schema migration from version 0 to version 1."
@removeMessageClockFromDatabase()
.then @removeMessageClockFromMessageFolderNames
.then @removeMessageClockFromMessageFiles
.then =>
console.log '✅ Version 0 to 1: data schema migration complete.'
removeMessageClockFromMessageFolderNames: =>
console.log 'Promising to remove message clocks from message folder names.'
numberOfFoldersAffected = 0
walk.folders @syncFolder
.then (folders) =>
Promise.series folders, (folder) =>
matches = folder.match @depracatedMessageClockMatcher
if matches != null
newFolder = folder.replace @depracatedMessageClockMatcher, ''
fs.moveAsync folder, newFolder
.then =>
numberOfFoldersAffected++
.then =>
console.log "✅ Updated #{numberOfFoldersAffected} message folder names out of #{folders.length} folders."
removeMessageClockFromMessageFiles: =>
console.log 'Promising to remove message clocks from message files.'
numberOfMessageFilesAffected = 0
walk.files @syncFolder
.then (files) =>
Promise.series files, (file) =>
if file.substr(-10) == 'index.html'
fs.readFileAsync file, {encoding: 'utf8'}
.then (message) =>
matches = message.match @depracatedMessageClockMatcher
if matches != null
message = message.replace @depracatedMessageClockMatcher, ''
fs.writeFileAsync file, message
.then =>
# console.log "Removed message clock from message in #{file}."
numberOfMessageFilesAffected++
.then =>
console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."
removeMessageClockFromDatabase: =>
console.log 'Promising to remove message clocks from database.'
numberOfDatabaseKeysAffected = 0
return new Promise ((fulfill, reject) =>
# Create options for a new stream
options = { gt: '\x00', lt: '\uffff' }
dataStream = @db.plain.createReadStream options
dataStream.on 'error', (err) =>
console.log "Data stream error: #{err}"
toArray = WriteStream.toArray (data) =>
Promise.series data, (datum) =>
matches = datum.key.match @depracatedMessageClockMatcher
if matches != null
# Remove the message clock from message keys.
newKey = datum.key.replace @depracatedMessageClockMatcher, ''
# Remove the message clock from message values.
newValue = datum.value.replace @depracatedMessageClockMatcher, ''
# console.log "Migrating #{datum.key} to #{newKey}…"
# console.log "Value:"
# console.log datum.value
operations = [
{type: 'put', key: newKey, value: newValue},
{type: 'del', key: datum.key}
]
# Carry out the operations in a batch.
@db.plain.batch(operations)
.then =>
# console.log "OK."
numberOfDatabaseKeysAffected++
.catch (error) =>
console.log "Error while updating #{datum.key} to #{newKey}: #{error}"
.then =>
console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."
process.nextTick(=>
console.log "Database migration complete."
fulfill true
)
.catch (error) =>
console.log "Could not migrate the database: #{error}."
dataStream.pipe toArray
)
module.exports = Migrations
\ No newline at end of file
......@@ -28,7 +28,7 @@ WriteStream = require 'write-stream'
liveStream = require 'level-live-stream'
database = require './database'
database = (require './database').withSublevels
class StreamWeaver
......@@ -209,7 +209,7 @@ class StreamWeaver
#
# Also known as document/blog order. All other
# timelines are displayed in conversation order (latest post last).
options.reverse = !(timeline == @timeline.public || timeline == @timeline.private || timeline == @timeline.allFriends)
options.reverse = (timeline == @timeline.public || timeline == @timeline.private || timeline == @timeline.allFriends)
# console.log "TIMELINE REVERSED? #{options.reverse}"
# console.log "@timelines[timeline] = #{@timelines[timeline]}"
......
################################################################################
##########################################################################################
#
# Ind.ie Heartbeat Node
#
# Database — singleton access to the LevelDB database.
#
# This is a singleton object that proxies a reference to the database instance.
# This is a singleton object that proxies references to the LevelDB database.
#
# Usage:
#
# database = require './database'
#
# database.plain # Reference to plain LevelUp database
# database.withSublevels # level-sublevel-wrapped reference
# database.withPromises # LevelUp promisified with Thrush
# database.withSublevelsAndPromises # Wrapped in level-sublevel & promisified with Thrush
#
# This is independent technology. See ind.ie/manifesto
#
......@@ -19,16 +28,20 @@
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
##########################################################################################
path = require 'path-extra'
LevelUp = require 'levelup'
Sublevel = require 'level-sublevel'
LevelUpThrushPromisify = require './levelup-thrush-promisify'
class Database
instance = null
db: null
plain: null
withSublevels: null
withPromises: null
withSublevelsAndPromises: null
constructor: ->
......@@ -46,7 +59,11 @@ class Database
databasePath = 'db'
console.log "Waystone LevelDB database path: #{databasePath}"
@db = Sublevel (LevelUp databasePath)
@plain = LevelUp databasePath
@withSublevels = Sublevel @plain
@withPromises = LevelUpThrushPromisify @plain
@withSublevelsAndPromises = LevelUpThrushPromisify @withSublevels
# Save reference to the singleton instance.
instance = @
......@@ -56,4 +73,4 @@ class Database
#
# Actually proxy the database instance to cut down on boilerploit code when used.
#
module.exports = (new Database).db
module.exports = new Database
/*
Promisifies LevelUp
Pass either a LevelUp instance or a level-sublevel instance of the database.
Based on Level Bluebird Promise by Alex Ferreira
(Original: https://github.com/alexferreira/level-bluebird-promise)
Copyright © Aral Balkan.
*/
var _ = require('lodash')
, Promise = require('thrush')
, Manifest = require('level-manifest')
module.exports = function (db) {
manifest = new Manifest(db);
return _resursive(db, manifest);
};
function _resursive(db, manifest){
_.chain(manifest.methods).pick(function(obj){
return obj.type == 'async';
}).each(function(v, k){
if (db[k] != undefined)
{
return db[k] = Promise.promisify(db[k]);
}
}).value();
var sublevels = manifest.sublevels || {};
for (var name in sublevels) if (_.has(sublevels, name)){
_resursive(db.sublevels[name], sublevels[name]);
}
if (_.isFunction(db.sublevel)) {
var Sub = db.sublevel;
db.sublevel = function(name) {
var sublevel = Sub.apply(this, arguments);
if (!_.has(sublevels, name)) _resursive(sublevel, new Manifest(sublevel));
return sublevel;
}
}
return db;
};
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
Promise = require 'thrush'
LevelUp = require 'levelup'
LevelPromisify = require 'level-promisify'
WriteStream = require 'write-stream'
fs = Promise.promisifyAll(require 'fs-extra')
walk = require './walk'
#level = LevelUp 'db'
# LOCAL TEST
level = LevelUp '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/db'
db = LevelPromisify level
#
# Version 0 to version 1.
#
# Removes the local message clocks from database keys and from message folder names.
#
depracatedMessageClockMatcher = /\d{9}-/g
version0toVersion1 = ->
removeMessageClockFromDatabase()
.then removeMessageClockFromMessageFolderNames
.then removeMessageClockFromMessageFiles
.then ->
console.log '✅ Version 0 to 1: data schema migration complete.'
removeMessageClockFromMessageFolderNames = ->
console.log 'Promising to remove message clocks from message folder names.'
numberOfFoldersAffected = 0
(walk.folders '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
matches = file.match depracatedMessageClockMatcher
if matches != null
newFile = file.replace depracatedMessageClockMatcher, ''
# console.log "From: #{file}\nTo: #{newFile}\n\n"
fs.moveAsync file, newFile
.then ->
# console.log "Moved #{file} to #{newFile}"
numberOfFoldersAffected++
.then ->
console.log "✅ Migrated #{numberOfFoldersAffected} folder names out of #{files.length} files."
removeMessageClockFromMessageFiles = ->
console.log 'Promising to remove message clocks from message files.'
numberOfMessageFilesAffected = 0
(walk.files '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
if file.substr(-10) == 'index.html'
fs.readFileAsync file, {encoding: 'utf8'}
.then (message) ->
console.log "Checking #{file}"
matches = message.match depracatedMessageClockMatcher
if matches != null
message = message.replace depracatedMessageClockMatcher, ''
fs.writeFileAsync file, message
.then ->
console.log "Removed message clock from message in #{file}."
numberOfMessageFilesAffected++
.then ->
console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."
removeMessageClockFromDatabase = ->
console.log 'Promising to remove message clocks from database.'
numberOfDatabaseKeysAffected = 0
return new Promise ((fulfill, reject) ->
# Create options for a new stream
options = { gt: '\x00', lt: '\uffff' }
dataStream = db.root.createReadStream options
dataStream.on 'error', (err) ->
console.log "Data stream error: #{err}"
toArray = WriteStream.toArray (data) ->
Promise.series data, (datum) ->
matches = datum.key.match depracatedMessageClockMatcher
if matches != null
# Remove the message clock from message keys.
newKey = datum.key.replace depracatedMessageClockMatcher, ''
# Remove the message clock from message values.
newValue = datum.value.replace depracatedMessageClockMatcher, ''
# console.log "Migrating #{datum.key} to #{newKey}…"
# console.log "Value:"
# console.log datum.value
operations = [
{type: 'put', key: newKey, value: newValue},
{type: 'del', key: datum.key}
]
# Carry out the operations in a batch.
db.batch(operations)
.then ->
# console.log "OK."
numberOfDatabaseKeysAffected++
.catch (error) ->
console.log "Error while updating #{datum.key} to #{newKey}: #{error}"
.then ->
console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."
process.nextTick(->
console.log "Database migration complete."
fulfill true
)
.catch (error) ->
console.log "Could not migrate the database: #{error}."
dataStream.pipe toArray
)
# TEST
version0toVersion1()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment