Ind.ie is now Small Technology Foundation.
Migrations.coffee 8.58 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
################################################################################
#
# Ind.ie Heartbeat Node
#
# Data schema migrations.
#
# Migrates the data schemas used between versions.
#
# This is independent technology. See ind.ie/manifesto
#
# Copyright © Aral Balkan. Copyright © Ind.ie. All Rights Reserved.
#
# We are working on releasing the Heartbeat code under a license that is free
# as in freedom and yet allows us to publish and distribute Heartbeat on the
# App Store. In the meanwhile, we are releasing it, all rights reserved,
# for review.
#
# To follow the updates on the licensing, please see the
# Towards Ind.ie Commons Licenses thread on the Ind.ie forum
# (https://forum.ind.ie/t/towards-ind-ie-commons-licenses/690)
#
################################################################################

Promise = require 'thrush'
WriteStream = require 'write-stream'
path = require 'path-extra'
27
fs = require 'fs-extra-as-promised'
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84

targz = require 'tar.gz'
walk = require './walk'

class Migrations
	instance = null

	db: require './database'

	homeFolder: null
	syncFolder: null
	databaseFolder: null
	dataSchemaVersion: null
	meta: null
	dbFolderSnapshotPath: null
	syncFolderSnapshotPath: null
	dataSchemaVersionInDatabase: 0

	constructor: (config) ->

		# Singleton access
		if instance
			return instance

		#
		# Initialise the singleton instance
		#

		console.log 'Initialising Migrations singleton.'

		@homeFolder = config.homeDirectory
		@syncFolder = config.syncFolder
		@databaseFolder = config.databaseFolder
		@dataSchemaVersion = config.dataSchemaVersion

		console.log "CURRENT Data schema version: #{@dataSchemaVersion}"

		return instance


	runMigrations: () =>
		# Using LevelUp directly as level-sublevel can’t create a read stream on the root
		# that iterates into the child sublevels.
		@meta = @db.withSublevels.sublevel 'meta'

		@meta.get 'dataSchemaVersion'
		.then (dataSchemaVersionInDatabase) =>
			console.log "Data schema version in database: #{dataSchemaVersionInDatabase}"
			# So much for raising an error if a key doesn’t exist :)
			if dataSchemaVersionInDatabase == ''
				@dataSchemaVersionInDatabase = 0
			else
				@dataSchemaVersionInDatabase = dataSchemaVersionInDatabase
		.catch (error) =>
			# Not really an error; person just has earlier version
			# of Heartbeat than the first one where we implemented migrations.
			console.log "No data schema version in database yet. (Version 0)."
85

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
		.finally =>
			if "#{@dataSchemaVersionInDatabase}" == "#{@dataSchemaVersion}"
				console.log "Heartbeat data schema version is up to date (#{@dataSchemaVersionInDatabase}). Not running migrations."
				return
			else
				#
				# Backup the current configuration so we can revert to it if something goes wrong.
				#

				@dbFolderSnapshotPath = path.join @homeFolder, "db-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"
				@syncFolderSnapshotPath = path.join @homeFolder, "Sync-snapshot-v#{@dataSchemaVersionInDatabase}.tar.gz"

				targz().compress @databaseFolder, @dbFolderSnapshotPath
				.then =>
					targz().compress @syncFolder, @syncFolderSnapshotPath
					.then =>
						console.log "Current configuration snapshot complete."

						#
						# Run migrations
						#
						console.log ">>> #{@dataSchemaVersionInDatabase}"
						console.log "Running migrations: #{[@dataSchemaVersionInDatabase...@dataSchemaVersion]}"

						Promise.series [@dataSchemaVersionInDatabase...@dataSchemaVersion], (migrationVersion) =>
							# Note migration versions are always numbered in sequence (0,1,2,3…)
							# If the app is on data schema version 0 and the current version is 3, it will go through 0->1, 1->2, 2->3.
							console.log "Running migration from version #{migrationVersion} to #{migrationVersion+1}"
							console.log "Selector: migrateVersion#{migrationVersion}"
							@["migrateVersion#{migrationVersion}"]()
						.then =>
							@meta.put 'dataSchemaVersion', @dataSchemaVersion
							.then =>
								console.log "Migrations complete. Updated app’s data schema version to #{@dataSchemaVersion}."
						.catch (error) =>
							# Migrations failed.

							# Restore the initial configuration from our snapshot.
							console.log "Migrations failed. Restoring initial configuration from snapshot. #{error}"
							targz().extract @dbFolderSnapshotPath, @databaseFolder
							.then =>
								targz().extract @syncFolderSnapshotPath, @syncFolder
								.then =>
									console.log "Snapshot restored for version #{@dataSchemaVersionInDatabase}."

									# Bail — we cannot migrate the data schema to the latest version. We can’t recover from this.
									# Person either has to use the older version or file a bug report.
									throw new Error "Migrations failed. Cannot continue."
134

135 136 137 138
					.catch (error) =>
						# Backing up the sync folders failed. Bail.
						# Not going to risk messing up the person’s current configuration / data.
						throw new Error "Sync folder migration snapshot failed. Cannot continue."
139

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
				.catch (error) =>
					# Backing up the database folders failed. Bail.
					# Not going to risk messing up the person’s current configuration / data.
					throw new Error "Database migration snapshot failed. Cannot continue."

	#
	# Version 0 to version 1.
	#
	# Removes the local message clocks from database keys and from message folder names.
	#

	depracatedMessageClockMatcher: /\d{9}-/g

	migrateVersion0: =>
		console.log "Running data schema migration from version 0 to version 1."

		@removeMessageClockFromDatabase()
		.then @removeMessageClockFromMessageFolderNames
		.then @removeMessageClockFromMessageFiles
		.then =>
			console.log '✅ Version 0 to 1: data schema migration complete.'


	removeMessageClockFromMessageFolderNames: =>

		console.log 'Promising to remove message clocks from message folder names.'

		numberOfFoldersAffected = 0

		walk.folders @syncFolder
		.then (folders) =>

			Promise.series folders, (folder) =>

				matches = folder.match @depracatedMessageClockMatcher
				if matches != null
					newFolder = folder.replace @depracatedMessageClockMatcher, ''

					fs.moveAsync folder, newFolder
					.then =>
						numberOfFoldersAffected++
			.then =>
				console.log "✅ Updated #{numberOfFoldersAffected} message folder names out of #{folders.length} folders."


	removeMessageClockFromMessageFiles: =>

		console.log 'Promising to remove message clocks from message files.'

		numberOfMessageFilesAffected = 0

		walk.files @syncFolder
		.then (files) =>

			Promise.series files, (file) =>

				if file.substr(-10) == 'index.html'
					fs.readFileAsync file, {encoding: 'utf8'}

					.then (message) =>

						matches = message.match @depracatedMessageClockMatcher
						if matches != null

							message = message.replace @depracatedMessageClockMatcher, ''

							fs.writeFileAsync file, message
							.then =>
								# console.log "Removed message clock from message in #{file}."
								numberOfMessageFilesAffected++
			.then =>
				console.log "✅ Migrated #{numberOfMessageFilesAffected} messages out of #{files.length} files."


	removeMessageClockFromDatabase: =>

		console.log 'Promising to remove message clocks from database.'

		numberOfDatabaseKeysAffected = 0

		return new Promise ((fulfill, reject) =>

			# Create options for a new stream
			options = { gt: '\x00', lt: '\uffff' }

			dataStream = @db.plain.createReadStream options

			dataStream.on 'error', (err) =>
				console.log "Data stream error: #{err}"

			toArray = WriteStream.toArray (data) =>

				Promise.series data, (datum) =>

					matches = datum.key.match @depracatedMessageClockMatcher
					if matches != null

						# Remove the message clock from message keys.
						newKey = datum.key.replace @depracatedMessageClockMatcher, ''

						# Remove the message clock from message values.
						newValue = datum.value.replace @depracatedMessageClockMatcher, ''

						# console.log "Migrating #{datum.key} to #{newKey}…"
						# console.log "Value:"
						# console.log datum.value

						operations = [
							{type: 'put', key: newKey, value: newValue},
							{type: 'del', key: datum.key}
						]

						# Carry out the operations in a batch.
						@db.plain.batch(operations)
						.then =>
							# console.log "OK."
							numberOfDatabaseKeysAffected++
						.catch (error) =>
							console.log "Error while updating #{datum.key} to #{newKey}: #{error}"

				.then =>

					console.log "✅ Migrated #{numberOfDatabaseKeysAffected} of #{data.length} keys."

					process.nextTick(=>
						console.log "Database migration complete."
						fulfill true
					)

				.catch (error) =>
					console.log "Could not migrate the database: #{error}."

			dataStream.pipe toArray
		)

module.exports = Migrations