Commit 59e87d52 authored by Aral Balkan's avatar Aral Balkan
Browse files

Organised into classes. Added Dokku deployment Procfile. Added install and...

Organised into classes. Added Dokku deployment Procfile. Added install and deploy scripts and license.
parent e455043b
content
.DS_Store
node_modules
################################################################################
#
# App data.
#
# This is Independent Technology. See https://ind.ie/manifesto
#
# Copyright © 2015, Aral Balkan. © 2014-2015, Ind.ie.
# Released with ♥ by Ind.ie under GNU AGPLv3 or later.
# Free as in freedom. Please see the LICENSE file.
#
################################################################################
path = require 'path-extra'
class App
instance = null
identifier: null
homeDirectory: null
constructor: ->
# Singleton access.
if instance
return instance
# Initialise global properties of the app.
@identifier = 'ind.ie.blockdown-builder'
@homeDirectory = path.join path.homedir(), @identifier
# Store and return reference to singleton instance.
instance = @
return instance
# Returns a reference to the App singleton.
module.exports = App
fs = require 'fs-extra-as-promised'
marked = require 'marked'
jsonlint = require 'jsonlint'
chai = require 'chai'
expect = chai.expect
assert = chai.assert
chai.should()
Promise = require 'thrush'
glob = require 'glob'
globAsync = Promise.promisify glob
eyespect = require 'eyespect'
winston = require 'winston'
app = require './App'
ContentBlockerRule = require './ContentBlockerRule'
# Removes whitespace from the start and end of a string.
trim = (string) ->
string.replace /^\s+|\s+$/g, ''
class Blockdown
markdownRenderer: null # (marked.Renderer)
blockdownRenderer: null # (marked.Renderer)
rules: null # (Array)
#
# Public methods.
#
constructor: ->
# Initialise the rules array
@rules = []
# Initialise two renderers (one for regular Markdown, the
# other for the Blockdown code sections).
@markdownRenderer = new marked.Renderer()
@blockdownRenderer = new marked.Renderer()
@blockdownRenderer.code = @codeRenderer
# Set up logging.
logFile = path.join app.homeDirectory, 'Blockdown.log'
logger = new winston.Logger
transports:
[
new (winston.transports.Console)()
new (winston.transports.File)({ filename: logFile })
]
logger.extend @
@info "Blockdown initiated. Logs at #{logFile}."
#
# Render all content recursively starting at contentPath.
#
render: (contentPath) ->
@info "Rendering content at path #{contentPath}…"
(globAsync "#{contentPath}/**/*.md", {})
.map (file)->
@info "Reading blockdown file: #{file}"
fs.readFileAsync file, 'utf-8'
.then (content) ->
@info "Converting #{file} to JSON."
marked content, renderer: @blockdownRenderer, gfm: true
.then ->
@info "Done.\n"
@info "Rules:\n"
@info eyespect.inspect rules
#
# Private methods.
#
#
# Hook into the marked Markdown renderer for the Blockdown MSON code sections.
#
codeRenderer: (code, language) =>
# console.log "Language: #{language}, Code: >#{code}<"
originalCode = code
if language == 'mson'
# Convert blocker rule MSON to JSON
# Do a very strict conversion of the MSON based on the block list JSON specification.
# Trim any leading or trailing whitespace
code = trim code
# Flatten the key/value pairs
safeDelimeter = '\udbff\udfff'
code = code.replace /^[\t, ]*-[\t, ]*(.*?):[\t, ]*(.*?)$/mg, "$1#{safeDelimeter}$2"
# Split into lines
lines = code.split "\n"
#
# Create the rule.
#
rule = new ContentBlockerRule
lines.forEach (line) ->
keyValuePair = line.split safeDelimeter
key = keyValuePair[0]
value = keyValuePair[1]
rule[key](value)
# Lint the content blocker JSON rule string
try
@lintRule rule.value()
catch e
console.log "Content blocker rule lint error, please check that your block rule MSON is valid.\n\n#{e}"
# TODO: Don’t exit — raise an error.
process.exit 1
# Add the linted rule to the rules for this Markdown document.
@rules.push rule.value()
# TODO: What is the effect of this fallthrough?
return @markdownRenderer.code originalCode, language
#
# Content blocker JSON rule sting linter.
#
# All linting rules based on, and quoted from:
# https://www.webkit.org/blog/3476/content-blockers-first-look/
#
lintRule: (rule) ->
try
#
# Content blocker format
#
# The content blocker rules are passed in JSON format.
# The top level object is an array containing every rule that needs to be loaded.
#
# Each rule of the content blocker is a dictionary with two parts:
# a trigger which activates the rule, and an action defining what to do when the rule is activated.
#
expect(rule, 'rule').to.have.property 'trigger'
expect(rule, 'rule').to.have.property 'action'
#
# The “trigger” defines what properties activate a rule. When the rule is activated, its action is
# queued for execution. When all the triggers have been evaluated, the actions are applied in order.
#
# The valid fields in the trigger are:
#
# “url-filter” (string, mandatory): matches the resource’s URL.
# “url-filter-is-case-sensitive”: (boolean, optional): changes the “url-filter” case-sensitivity.
# “resource-type”: (array of strings, optional): matches how the resource will be used.
# “load-type”: (array of strings, optional): matches the relation to the main resource.
# “if-domain”/”unless-domain” (array of strings, optional): matches the domain of the document.
# assert.equal typeof trigger['url-filter'] == 'string'
#
trigger = rule['trigger']
triggerProperties = new Set Object.keys(rule['trigger'])
# Check for existence of mandatory url-filter property.
expect(trigger, 'trigger').to.have.property 'url-filter'
urlFilter = trigger['url-filter']
#
# The Regular expression format
#
# It is possible to use the beginning of line (“^”) and end of line (“$”) marker but they are restricted
# to be the first and last character of the expression. For example, a pattern like “^bar$” is perfectly valid,
# while “(foo)?^bar$” causes a syntax error.
#
# URL Filter Regular Expression special case: ^
indexOfCaret = urlFilter.indexOf '^'
theIndexOfTheFirstCharacter = 0
if indexOfCaret > -1 then expect(indexOfCaret, 'If ^ is used in the URL filter regular expression, it must be the first character').to.equal theIndexOfTheFirstCharacter
# URL Filter Regular Expression special case: $
indexOf$ = urlFilter.indexOf '$'
theIndexOfTheLastCharacter = urlFilter.length-1
if indexOf$ > -1 then expect(indexOf$, 'If $ is used in the URL filter regular expression, it must be the last character').to.equal theIndexOfTheLastCharacter
#
# All URL matching is done against the canonical version of the URL. As such, you can expect the URL to be
# completely ASCII. The domain will already be punycode encoded. Both the scheme and domain are already
# lowercase. The resource part of the URL is already percent encoded.
#
# Since the URL is known to be ASCII, the url-filter is also restricted to ASCII. Patterns with non-ASCII
# characters result in a parse error.
#
urlFilterIsASCII = /^[\x00-\x7F]*$/.test urlFilter
urlFilterIsASCII.should.be.true
#
# Check trigger types
#
urlFilter.should.be.a 'string'
if triggerProperties.has 'url-filter-is-case-sensitive'
trigger['url-filter-is-case-sensitive'].should.be.a 'boolean', 'url-filter-is-case-sensitive'
#
# The optional field “resource-type” specifies the type of load to match.
# The content of this field is an array with all the types of load that can activate the trigger.
#
# The possible values are:
#
# “document”
# “image”
# “style-sheet”
# “script”
# “font”
# “raw” (any untyped load, like XMLHttpRequest)
# “svg-document”
# “media”
# “popup”
#
if triggerProperties.has 'resource-type'
resourceTypes = trigger['resource-type']
@assertIsAnArrayOfStrings resourceTypes, 'trigger', 'resource-type'
@assertActualValuesAreValid resourceTypes, ['document', 'image', 'style-sheet', 'script', 'font', 'raw', 'svg-document', 'media', 'popup'], 'trigger', 'resource-type'
#
# The field “load-type” defines the relation between the domain of the resource being loaded
# and the domain of the document. The two possible values are:
#
# “first-party”
# “third-party”
#
if triggerProperties.has 'load-type'
loadTypes = trigger['load-type']
@assertIsAnArrayOfStrings loadTypes, 'trigger', 'load-type'
@assertActualValuesAreValid loadTypes, ['first-party', 'third-party'], 'trigger', 'load-type'
if triggerProperties.has 'if-domain'
@assertIsAnArrayOfStrings trigger['if-domain'], 'trigger', 'if-domain'
if triggerProperties.has 'unless-domain'
@assertIsAnArrayOfStrings trigger['unless-domain'], 'trigger', 'unless-domain'
# Make sure both if-domain and unless-domain are not present at the same time.
assert.equal (triggerProperties.has 'if-domain' and triggerProperties.has 'unless-domain'), false, 'Both if-domain and unless-domain should not be set at the same time on the rule’s trigger.'
# Check for invalid Trigger properties.
@assertActualValuesAreValid triggerProperties, ['url-filter', 'url-filter-is-case-sensitive', 'resource-type', 'load-type', 'if-domain', 'unless-domain'], 'trigger', 'property'
#
# The “action” part of the dictionary defines what the engine should do
# when a resource is matched by a trigger.
#
# Currently, the action object has only 2 valid fields:
#
# * “type” (string, mandatory): defines what to do when the rule is activated.
# * “selector” (string, mandatory for the “css-display-none” type):
# defines a selector list to apply on the page.
#
action = rule['action']
actionProperties = new Set Object.keys(rule['action'])
expect(actionProperties.has 'type', 'Action property ‘type’ is mandatory and should exist.').to.be.true
actionType = action['type']
expect(actionType, 'action.type').to.be.a 'string'
if actionType == 'css-display-none'
expect(actionProperties.has 'selector', 'If the action type is ‘css-display-none’ then action property ‘selector’ must exist.').to.be.true
actionSelector = action['selector']
expect(actionSelector, 'action.selector').to.be.a 'string'
#
# There are 3 types of actions that limit resources:
#
# “block”, “block-cookies”, “css-display-none”.
#
# There is an additional type that does not have any impact on the resource
# but changes how the content extension behaves: “ignore-previous-rules”.
#
assert.equal (actionType == 'block' or actionType == 'block-cookies' or actionType == 'css-display-none' or actionType == 'ignore-previous-rules'), true, 'Action type should be valid.'
catch e
throw new Error "#{e} #{eyespect.inspect rule}"
#
# Checks that the passed object is an array of strings. Throws if not.
#
assertIsAnArrayOfStrings: (obj, parentObjectsHumanName, propertysHumanName) ->
expect(obj, "The value of #{parentObjectsHumanName}’s #{propertysHumanName} property should be an array.").to.be.an 'array'
obj.forEach (property) ->
expect(property, "The value of #{parentObjectsHumanName}’s #{propertysHumanName} property should be an array of strings.").to.be.a 'string'
#
# Checks that the values in the actual values array conform to the set of values in the valid values array.
# Throws if not.
#
assertActualValuesAreValid: (actualValuesArray, validValuesArray, parentObjectsHumanName, propertysHumanName) ->
actualValues = new Set actualValuesArray
validValues = new Set validValuesArray
# Perform a union between the set of valid resource type values and the actual resource type values
# and check that there are no extra values (i.e., invalid values) in the resulting set.
validValues.forEach (validValue) -> actualValues.add validValue
expect(validValues.size, "All #{parentObjectsHumanName} #{propertysHumanName} values should be valid.").to.equal actualValues.size
module.exports = Blockdown
################################################################################
#
# Transpiles a Blockdown rule into Webkit Content Blocker Rule.
#
# This is Independent Technology. See https://ind.ie/manifesto
#
# Copyright © 2015, Aral Balkan. © 2014-2015, Ind.ie.
# Released with ♥ by Ind.ie under GNU AGPLv3 or later.
# Free as in freedom. Please see the LICENSE file.
#
################################################################################
class ContentBlockerRule
rule: null
constructor: ->
@rule =
trigger: {}
action: {}
#
# Helper methods.
#
arrayOfStringsFromCommaSeparatedString: (string) ->
values = []
(string.split ',').forEach (value) ->
values.push (trim value)
values
#
# API
#
asJSONString: => JSON.stringify @rule
value: => @rule
#
# Transpiler methods.
#
# Trigger:
'trigger': () -> ''
'url-filter': (value) => @rule.trigger['url-filter'] = value
'url-filter-is-case-sensitive': (value) -> @rule.trigger['url-filter-is-case-sensitive'] = JSON.parse value
'resource-type': (value) => @rule.trigger['resource-type'] = @arrayOfStringsFromCommaSeparatedString value
'load-type': (value) => @rule.trigger['load-type'] = @arrayOfStringsFromCommaSeparatedString value
'if-domain': (value) => @rule.trigger['if-domain'] = @arrayOfStringsFromCommaSeparatedString value
'unless-domain': (value) => @rule.trigger['unless-domain'] = @arrayOfStringsFromCommaSeparatedString value
# Action:
'action': () -> ''
'type': (value) => @rule.action['type'] = value
'selector': (value) => @rule.action['selector'] = value
module.exports = ContentBlockerRule
################################################################################
#
# Listens for deployment web hook from Gitlab, pulls the latest changes,
# and runs the Blockdown builder.
#
# With thanks to Mike Devita’s gitlab-webhook
# https://github.com/mikedevita/gitlab-webhook/blob/master/server.js
#
# This is Independent Technology. See https://ind.ie/manifesto
#
# Copyright © 2015, Aral Balkan. © 2014-2015, Ind.ie.
# Released with ♥ by Ind.ie under GNU AGPLv3 or later.
# Free as in freedom. Please see the LICENSE file.
#
################################################################################
express = require 'express'
http = require 'http'
bodyParser = require('body-parser')
exec = require('child_process').exec
path = require 'path-extra'
winston = require 'winston'
Blockdown = require './Blockdown'
App = require './App'
class GitlabWebhookServer
instance = null
app:null
server:null
config:null
constructor: ->
# Singleton access.
if instance
return instance
# Set up logging.
logFile = path.join (new App).homeDirectory, 'GitlabWebhookServer.log'
logger = new winston.Logger
transports:
[
new (winston.transports.Console)()
new (winston.transports.File)({ filename: logFile })
]
logger.extend @
@info "GitlabWebhookServer session started. Logs at #{logFile}."
# Make sure you’ve copied config.coffee to your home folder and configured it.
@config = require (path.join (new App).homeDirectory, 'config')
# Start the server.
@start()
# Save reference to the singleton instance.
instance = @
return instance
start: =>
@info 'Starting the Gitlab webhook server…'
#
# Set up Express
#
app = express()
# app.set 'view engine', 'html'
# app.set 'views', __dirname + '/views'
# app.set 'config', @config
# app.use express.static('./static/')
app.use bodyParser.json()
app.use bodyParser.urlencoded {extended: true}
#
# Routes
#
app.post '/build/:token', (request, response) =>
if typeof request.params.token == 'undefined' || request.params.token != @config.token
response.status(403).send '403 not authorized'
else
gitPullCommand = "cd #{@config.contentPath} && git pull origin master"
exec gitPullCommand, (error, stdout, stderr) ->
if error
@error stderr
response.status(500).send stderr
else
@info stdout
blockdown = new Blockdown
blockdown.render @config.contentPath .then res.status(200).send stdout
#
# Create the server
#
port = process.env.PORT || 8888
@server = http.createServer app
@server.listen port, () =>
@info "Gitlab Webhook Server runnning at http://localhost:#{port}"
@app = app
module.exports = GitlabWebhookServer
This diff is collapsed.
################################################################################
#
# Builds the Blockdown data for the iOS/Mac apps and for the Blockdown site.
#
# This is Independent Technology. See https://ind.ie/manifesto
#
# Copyright © 2015, Aral Balkan. © 2014-2015, Ind.ie.
# Released with ♥ by Ind.ie under GNU AGPLv3 or later.
# Free as in freedom. Please see the LICENSE file.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
GitlabWebhookServer = require './GitlabWebhookServer'
new GitlabWebhookServer
web: node_modules/forever/bin/forever -c coffee index.coffee
\ No newline at end of file
#
# Configuration file.
#
# Note: copy this file to ~/ind.ie.blockdown-builder/config.coffee and customise the values, below.
#
module.exports =
token: 'random-token-key' # Generate and use a random token key
contentPath: '/path/to/your/Blockdown/content/repository'
git push live master
\ No newline at end of file
fs = require 'fs-extra-as-promised'
marked = require 'marked'
jsonlint = require 'jsonlint'
#should = require 'should'
chai = require 'chai'
expect = chai.expect
assert = chai.assert
chai.should()
Promise = require 'thrush'
glob = require 'glob'
globAsync = Promise.promisify glob
eyespect = require 'eyespect'
# Courtesy: http://stackoverflow.com/a/10941797
Object::extend = (objects...) ->
for object in objects
for key, value of object
@[key] = value
return
regularRenderer = new marked.Renderer()
renderer = new marked.Renderer()
# TEST: Read a Blockopedia Markdown file
# content = fs.readFileSync 'content/facebook.md', 'utf-8'
rules = []
# Removes whitespace from the start and end of a string.
trim = (string) ->
string.replace /^\s+|\s+$/g, ''
</