Ind.ie is now Small Technology Foundation.
Commit e720341f authored by Aral Balkan's avatar Aral Balkan

Migration script now also removing message clock from message contents and folder names.

parent 8e87ee36
......@@ -299,7 +299,7 @@ class Main
#
# Starts all of the sevices required for Heartbeat to function in
# Starts all of the sevices required for Heartbeat to function in
# its main messaging mode (as opposed to in account setup mode).
#
startAll: =>
......@@ -417,7 +417,7 @@ class Main
console.log "ℹ️ Waystone Device ID: #{waystoneDeviceID}"
setInterval =>
# console.log 'ℹ️ Running on local Waystone, going to send a discovery hint to Pulse.'
# console.log 'ℹ️ Running on local Waystone, going to send a discovery hint to Pulse.'
# console.log "ℹ️ Waystone Device ID: #{waystoneDeviceID}"
command = "curl -s -X POST --header \"X-API-Key: #{@pulseAPIKey}\" http://127.0.0.1:8080/rest/discovery/hint?device=#{waystoneDeviceID}\\&addr=192.168.59.103:22000"
......@@ -457,7 +457,7 @@ class Main
nodeSocketDidReceiveShutdown: =>
# Note that the following log statements will not be shown in the
# native client (as it will have exited by then). They’re here
# native client (as it will have exited by then). They’re here
# just to aid in possible future debugging.
winston.info 'Node: Info: Node socket did receive shutdown.'
winston.info 'Node: Info: Shutting down pulse process…'
......@@ -473,7 +473,7 @@ class Main
@pulseProcess?.stop()
@bonjourPublicWebServerServiceAnnouncement?.stop()
@ws?.close()
@ws?.close()
winston.info 'Cleaned up.'
......
################################################################################
#
# Ind.ie Heartbeat Node
# Ind.ie Heartbeat Node
#
# Database — singleton access to the LevelDB database.
# Database — singleton access to the LevelDB database.
#
# This is a singleton object that proxies a reference to the database instance.
# This is a singleton object that proxies a reference to the database instance.
#
# This is independent technology. See ind.ie/manifesto
# This is independent technology. See ind.ie/manifesto
#
# Copyright © 2014-2015, Aral Balkan. © 2014-2015, Ind.ie.
# Released with ♥ by Ind.ie under GNU AGPLv3 or later.
# Free as in freedom. Please see the LICENSE file.
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
......
# LevelPromise = require 'level-promise'
Sublevel = require 'level-sublevel'
WriteStream = require 'write-stream'
liveStream = require 'level-live-stream'
path = require 'path-extra'
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################
Promise = require 'thrush'
LevelUp = require 'levelup'
LevelPromisify = require 'level-promisify'
WriteStream = require 'write-stream'
fs = Promise.promisifyAll(require 'fs-extra')
# TEST
walk = require './walk'
#level = LevelUp 'db'
# LOCAL TEST
level = LevelUp '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/db'
db = LevelPromisify level
#db = LevelUp 'db'
#
# Version 0 to version 1.
#
# Removes the local message clocks from database keys and from message folder names.
#
# console.log db
depracatedMessageClockMatcher = /\d{9}-/g
version0toVersion1 = ->
removeMessageClockFromDatabase()
.then removeMessageClockFromMessageFolderNames
.then removeMessageClockFromMessageFiles
.then ->
console.log '✅ Version 0 to 1: data schema migration complete.'
removeMessageClockFromMessageFolderNames = ->
console.log 'Promising to remove message clocks from message folder names.'
numberOfFoldersAffected = 0
(walk.folders '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
matches = file.match depracatedMessageClockMatcher
if matches != null
newFile = file.replace depracatedMessageClockMatcher, ''
# console.log "From: #{file}\nTo: #{newFile}\n\n"
fs.moveAsync file, newFile
.then ->
# console.log "Moved #{file} to #{newFile}"
numberOfFoldersAffected++
.then ->
console.log "✅ Migrated #{numberOfFoldersAffected} folder names out of #{files.length} files."
return new Promise ((fulfill, reject) =>
removeMessageClockFromMessageFiles = ->
console.log 'Promising to remove message clocks from message files.'
numberOfMessageFilesAffected = 0
(walk.files '/Users/aral/Library/Containers/ind.ie.Heartbeat/Data/Pulse/Sync')
.then (files) ->
Promise.series files, (file) ->
if file.substr(-10) == 'index.html'
fs.readFileAsync file, {encoding: 'utf8'}
.then (message) ->
console.log "Checking #{file}"
matches = message.match depracatedMessageClockMatcher
if matches != null
message = message.replace depracatedMessageClockMatcher, ''
fs.writeFileAsync file, message
.then ->
console.log "Removed message clock from message in #{file}."
numberOfMessageFilesAffected++
.then ->
console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."
removeMessageClockFromDatabase = ->
console.log 'Promising to remove message clocks from database.'
numberOfDatabaseKeysAffected = 0
return new Promise ((fulfill, reject) ->
# Create options for a new stream
options = { gt: '\x00', lt: '\uffff' }
......@@ -32,37 +124,42 @@ version0toVersion1 = ->
dataStream.on 'error', (err) ->
console.log "Data stream error: #{err}"
toArray = WriteStream.toArray (data) =>
toArray = WriteStream.toArray (data) ->
Promise.series data, (datum) ->
depracatedMessageClockMatcher = /.*?\!(\d{9}-)/
matches = datum.key.match depracatedMessageClockMatcher
if matches != null
# Remove the depracated message clock from message keys.
newKey = datum.key.replace matches[1], ''
# Remove the message clock from message keys.
newKey = datum.key.replace depracatedMessageClockMatcher, ''
console.log "Migrating #{datum.key} to #{newKey}…"
# Remove the message clock from message values.
newValue = datum.value.replace depracatedMessageClockMatcher, ''
# console.log "Migrating #{datum.key} to #{newKey}…"
# console.log "Value:"
# console.log datum.value
operations = [
{type: 'put', key: newKey, value: datum.value},
{type: 'put', key: newKey, value: newValue},
{type: 'del', key: datum.key}
]
# Carry out the operations in a batch.
db.batch(operations)
.then ->
console.log "OK."
# console.log "OK."
numberOfDatabaseKeysAffected++
.catch (error) ->
console.log "Error while updating #{datum.key} to #{newKey}: #{error}"
.then ->
console.log "Migrated all #{data.length} entries."
console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."
process.nextTick(->
console.log "Fullfilling promise."
console.log "Database migration complete."
fulfill true
)
......@@ -72,4 +169,5 @@ version0toVersion1 = ->
dataStream.pipe toArray
)
version0toVersion1()
\ No newline at end of file
# TEST
version0toVersion1()
fs = require 'fs-extra'
path = require 'path-extra'
Promise = require 'thrush'
readdirAsync = Promise.promisify fs.readdir
statAsync = Promise.promisify fs.stat
#
# Based on http://stackoverflow.com/a/28130661 by Phil Mander
#
walkFiles = (directory) ->
results = []
(readdirAsync directory).map (file) ->
file = path.join directory, file
statAsync file
.then (stat) ->
if stat.isFile()
return results.push file
walkFiles file
.then (filesInDir) ->
results = results.concat filesInDir
.then ->
results
walkFolders = (directory) ->
results = []
(readdirAsync directory).map (file) ->
file = path.join directory, file
statAsync file
.then (stat) ->
if stat.isFile()
return
results.push file
walkFolders file
.then (foldersInFolder) ->
results = results.concat foldersInFolder
.then ->
results
module.exports =
files: walkFiles
folders: walkFolders
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment