commit 8837b1e04ff1c8c1d36785bb5b445ef44c7ca694
Author: Edward Betts <edward@4angle.com>
Date: Mon Jan 30 10:22:05 2017 +0000
initial commit
diff --git a/autoapp.py b/autoapp.py
new file mode 100644
index 0000000..1778f8d
--- /dev/null
+++ b/autoapp.py
@@ -0,0 +1,2 @@
+from sourcing import create_app
+app = create_app('config.default')
diff --git a/create_db.py b/create_db.py
new file mode 100755
index 0000000..c9438fd
--- /dev/null
+++ b/create_db.py
@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+
+from sourcing.model import Base
+from sourcing.database import init_db, session
+from sourcing import create_app
+
+app = create_app('config.default')
+db_url = app.config['DB_URL']
+init_db(db_url)
+engine = session.get_bind()
+# Base.metadata.drop_all(engine)
+Base.metadata.create_all(engine)
diff --git a/run.py b/run.py
new file mode 100755
index 0000000..de5dd14
--- /dev/null
+++ b/run.py
@@ -0,0 +1,7 @@
+#!/usr/bin/python3
+
+from sourcing import create_app
+
+if __name__ == "__main__":
+ app = create_app('config.default')
+ app.run('0.0.0.0', debug=True)
diff --git a/sourcing/__init__.py b/sourcing/__init__.py
new file mode 100644
index 0000000..aa2e424
--- /dev/null
+++ b/sourcing/__init__.py
@@ -0,0 +1,12 @@
+from flask import Flask
+from . import database
+from . import view
+from .utils import display_datetime
+
+def create_app(config):
+ app = Flask(__name__)
+ app.config.from_object(config)
+ database.init_app(app)
+ view.init_app(app)
+ app.jinja_env.filters['datetime'] = display_datetime
+ return app
diff --git a/sourcing/database.py b/sourcing/database.py
new file mode 100644
index 0000000..fe92862
--- /dev/null
+++ b/sourcing/database.py
@@ -0,0 +1,18 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+session = scoped_session(sessionmaker())
+
+def init_db(db_url):
+ session.configure(bind=get_engine(db_url))
+
+def get_engine(db_url):
+ return create_engine(db_url, pool_recycle=3600)
+
+def init_app(app):
+ db_url = app.config['DB_URL']
+ session.configure(bind=get_engine(db_url))
+
+ @app.teardown_appcontext
+ def shutdown_session(exception=None):
+ session.remove()
diff --git a/sourcing/forms.py b/sourcing/forms.py
new file mode 100644
index 0000000..475df64
--- /dev/null
+++ b/sourcing/forms.py
@@ -0,0 +1,75 @@
+from flask_wtf import Form
+from wtforms.fields import StringField, PasswordField, BooleanField, HiddenField, TextAreaField, RadioField, FileField, IntegerField
+from wtforms.validators import InputRequired, Email, Length, ValidationError, Regexp, NoneOf, Optional
+from .model import User, LoginError, re_username, reserved_name, user_exists
+
+PASSWORD_LEN = 64
+EMAIL_LEN = 64
+
+class SignupForm(Form):
+ username = StringField('username',
+ [InputRequired(),
+ Regexp(re_username),
+ NoneOf(reserved_name, message='Not available.'),
+ Length(min=3, max=64)],
+ [lambda name: name and name.replace(' ', '_')])
+ email = StringField('e-mail address',
+ [InputRequired(), Email(),
+ Length(min=5, max=EMAIL_LEN)],
+ description="we never share your e-mail address")
+ password = PasswordField('password',
+ [InputRequired(), Length(min=4, max=PASSWORD_LEN)])
+
+ def validate_username(form, field):
+ if user_exists(User.username, field.data):
+ raise ValidationError('Not available')
+
+ def validate_email(form, field):
+ if user_exists(User.email, field.data):
+ raise ValidationError('In use by another account')
+
+class LoginForm(Form):
+ user_or_email = StringField('username or e-mail address',
+ [InputRequired(), Length(min=3, max=EMAIL_LEN)],
+ [lambda name: name and name.replace(' ', '_')])
+ password = PasswordField('password',
+ [InputRequired(), Length(max=PASSWORD_LEN)])
+ remember = BooleanField('stay logged in')
+ next = HiddenField('next')
+
+ def validate(self):
+ rv = Form.validate(self)
+ if not rv:
+ return False
+
+ try:
+ self.user = User.attempt_login(self.user_or_email.data,
+ self.password.data)
+ return True
+ except LoginError as e:
+ self.user_or_email.errors.append(e.msg)
+ return False
+
+class ForgotPasswordForm(Form):
+ username_or_email = StringField('username or e-mail address',
+ [InputRequired(), Length(max=EMAIL_LEN)])
+
+class AccountSettingsForm(Form):
+ full_name = StringField('full name', [Length(max=64)])
+
+class ChangePasswordForm(Form):
+ old_password = PasswordField('current password',
+ [InputRequired(), Length(max=PASSWORD_LEN)])
+ new_password = PasswordField('new password',
+ [InputRequired(), Length(max=PASSWORD_LEN)])
+
+class SourceDocForm(Form):
+ text = TextAreaField('text', [InputRequired()])
+ db_price_per_character = IntegerField('price per character', [Optional()])
+ db_document_price = IntegerField('document price', [Optional()])
+
+class ItemForm(Form):
+ text = TextAreaField('text', [InputRequired()])
+
+class UploadSourceDocForm(Form):
+ sourcedoc_file = FileField('SourceDoc', [Regexp(r'^[^/\\]+\.txt$')])
diff --git a/sourcing/model.py b/sourcing/model.py
new file mode 100644
index 0000000..d64d547
--- /dev/null
+++ b/sourcing/model.py
@@ -0,0 +1,234 @@
+from flask import url_for
+from .database import session
+from .parse import parse_link, parse_sourcedoc_facet, parse_span
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, ForeignKey
+from sqlalchemy.types import String, Unicode, Integer, DateTime, Boolean, UnicodeText, Enum
+from sqlalchemy import func
+from sqlalchemy.orm import relationship, validates, synonym
+from sqlalchemy.sql import exists
+from flask_login import UserMixin
+from werkzeug.security import generate_password_hash, check_password_hash
+from sqlalchemy.ext.hybrid import hybrid_property
+import re
+
+from hashids import Hashids
+
+doc_hashids = Hashids(min_length=8)
+
+Base = declarative_base()
+Base.query = session.query_property()
+
+# list of disallowed usernames - maybe this should be in the database
+reserved_name = ['root', 'admin', 'administrator', 'support', 'info',
+ 'test', 'tech', 'online', 'old', 'new', 'jobs', 'login', 'job', 'ipad'
+ 'iphone', 'javascript', 'script', 'host', 'mail', 'image', 'faq',
+ 'file', 'ftp', 'error', 'warning', 'the', 'assistance', 'maintenance',
+ 'controller', 'head', 'chief', 'anon']
+
+re_username = re.compile('^\w+$', re.U)
+re_full_name = re.compile('^([-.\'" ]|[^\W\d_])+$', re.U)
+
+def user_exists(field, value):
+ return session.query(exists().where(field == value)).scalar()
+
+class TimeStampedModel(Base):
+ __abstract__ = True
+ created = Column(DateTime, default=func.now())
+ modified = Column(DateTime, default=func.now(), onupdate=func.now())
+
+class LoginError(Exception):
+ def __init__(self, msg):
+ self.msg = msg
+
+class User(TimeStampedModel, UserMixin):
+ __tablename__ = 'user'
+ id = Column(Integer, primary_key=True)
+ username = Column(Unicode(32), unique=True, nullable=False)
+ pw_hash = Column(String(160), nullable=False)
+ email = Column(Unicode(64), unique=True, nullable=False)
+ email_verified = Column(Boolean(), nullable=False, default=False)
+ disabled = Column(Boolean(), nullable=False, default=False)
+ deleted = Column(Boolean(), nullable=False, default=False)
+ is_super = Column(Boolean, nullable=False, default=False)
+ last_login = Column(DateTime)
+ full_name = Column(Unicode(64))
+ balance = Column(Integer, nullable=False, default=0)
+
+ user_id = synonym('id')
+ name = synonym('full_name')
+ user_name = synonym('username')
+
+ def __init__(self, **kwargs):
+ pw_hash = generate_password_hash(kwargs.pop('password'))
+ return super(User, self).__init__(pw_hash=pw_hash, **kwargs)
+
+ def __repr__(self):
+ return '<User: {!r}>'.format(self.username)
+
+ def set_password(self, password):
+ self.pw_hash = generate_password_hash(password)
+
+ def check_password(self, password):
+ return check_password_hash(self.pw_hash, password)
+
+ def get_id(self):
+ return self.id
+
+ @validates('email')
+ def validate_email(self, key, value):
+ assert '@' in value
+ return value
+
+ @validates('username')
+ def validate_usernane(self, key, value):
+ assert re_username.match(value)
+ return value
+
+ @validates('full_name')
+ def validate_full_name(self, key, value):
+ if value:
+ assert re_full_name.match(value)
+ return value
+
+ @hybrid_property
+ def is_live(self):
+ return self.email_verified & ~self.disabled & ~self.deleted
+
+ @classmethod
+ def lookup_user_or_email(cls, user_or_email):
+ field = cls.email if '@' in user_or_email else cls.username
+ return cls.query.filter(field == user_or_email).one_or_none()
+
+ @property
+ def mail_to_name(self):
+ '''Name to use on e-mails sent to the user.'''
+ return self.full_name or self.username
+
+ @classmethod
+ def attempt_login(cls, user_or_email, password):
+ user = cls.lookup_user_or_email(user_or_email)
+ if not user:
+ raise LoginError('user not found')
+ if user.disabled:
+ raise LoginError('user account disabled')
+ if not user.check_password(password):
+ raise LoginError('incorrect password')
+ return user
+
+class Item(TimeStampedModel):
+ __tablename__ = 'item'
+ id = Column(Integer, primary_key=True)
+ user_id = Column(Integer, ForeignKey('user.id'))
+ published = Column(DateTime)
+ type = Column(Enum('sourcedoc', 'xanadoc', 'xanalink', name='item_type'), nullable=False)
+ filename = Column(Unicode)
+ text = Column(UnicodeText)
+
+ user = relationship('User', backref='items')
+
+ __mapper_args__ = {
+ 'polymorphic_identity': 'item',
+ 'polymorphic_on': type,
+ }
+
+ @property
+ def hashid(self):
+ return doc_hashids.encode(self.id)
+
+ @classmethod
+ def get_by_hashid(cls, hashid):
+ return cls.query.get(doc_hashids.decode(hashid))
+
+ @property
+ def url(self):
+ return url_for('.view_item', username=self.user.username, hashid=self.hashid)
+
+ @property
+ def external_url(self):
+ return url_for('.view_item',
+ username=self.user.username,
+ hashid=self.hashid,
+ _external=True)
+
+ @property
+ def edit_url(self):
+ return url_for('.edit_item', username=self.user.username, hashid=self.hashid)
+
+ def title(self, titles=None):
+ if not titles:
+ titles = XanaLink.get_all_titles()
+ return self.type + ": " + titles.get(self, self.hashid)
+
+ @classmethod
+ def from_external(cls, url):
+ home = url_for('.home', _external=True)
+ if not url.startswith(home):
+ return
+ username, _, hashid = url[len(home):].partition('/')
+ q = cls.query.filter(User.username == username,
+ cls.id == doc_hashids.decode(hashid)[0])
+ return q.one_or_none()
+
+class XanaDoc(Item):
+ __tablename__ = 'xanadoc'
+ id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+
+ __mapper_args__ = {'polymorphic_identity': 'xanadoc'}
+
+class XanaLink(Item):
+ __tablename__ = 'xanalink'
+ id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+
+ __mapper_args__ = {'polymorphic_identity': 'xanalink'}
+
+ def parse(self):
+ return parse_link(self.text)
+
+ def title(self, titles=None):
+ if titles is None:
+ titles = XanaLink.get_all_titles()
+ if self in titles:
+ return self.type + ": " + titles[self]
+
+ parsed = self.parse()
+
+ if parsed['type'] == 'title':
+ ident = parsed['facets'][0][0].partition(': ')[2]
+ item = Item.from_external(ident)
+ if item in titles:
+ return parsed['type'] + " link for " + item.title(titles=titles)
+
+ return parsed['type'] + " link: " + self.hashid
+
+ @classmethod
+ def get_all_titles(cls):
+ titles = {}
+ for link in (obj.parse() for obj in cls.query):
+ if link['type'] != 'title':
+ continue
+ facet1, facet2 = link['facets']
+ link_type, _, ident = facet1[0].partition(': ')
+ item = Item.from_external(ident)
+ print(ident, item)
+
+ ident2, start, length = parse_span(facet2[0])
+ title = SourceDoc.from_external(ident2).text[start:length + start]
+ titles[item] = title
+ return titles
+
+class SourceDoc(Item):
+ __tablename__ = 'sourcedoc'
+ id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+ db_price_per_character = Column(Integer)
+ db_document_price = Column(Integer)
+
+ @property
+ def document_price(self):
+ return self.db_document_price or self.db_price_per_character * len(self.text)
+
+ @property
+ def price_per_character(self):
+ return self.db_price_per_character or self.db_document_price / len(self.text)
+
+ __mapper_args__ = {'polymorphic_identity': 'sourcedoc'}
diff --git a/sourcing/parse.py b/sourcing/parse.py
new file mode 100644
index 0000000..7cda3c3
--- /dev/null
+++ b/sourcing/parse.py
@@ -0,0 +1,273 @@
+import re
+import requests
+import os.path
+import random
+from html import escape
+from collections import defaultdict
+
+re_comment = re.compile(r'#.*')
+re_span_pointer = re.compile(r'span: (.*),start=(\d+),length=(\d+)')
+re_xanalink = re.compile('xanalink: +([^ ]+) *$')
+re_facet = re.compile('^facet\d* *=\s*(.*)\s*$')
+
+re_colon_slash = re.compile('[/:]+')
+
+project_dir = os.path.dirname(os.path.dirname(__file__))
+cache_location = os.path.join(project_dir, 'cache')
+
+max_sourcedoc_size = 600000
+
+xnb_per_char = 150000
+
+censor_urls = {'http://hyperland.com/xuCambDemo/J.Ineffable.txt',
+ 'http://royalty.pub/Ineffable/17/32/Trovato'}
+
+def censor_text(text):
+ return ''.join(chr(random.randint(9728, 9983)) if c.isalnum() else c
+ for c in text)
+
+def get_url(url):
+ filename = os.path.join(cache_location, url_filename(url))
+
+ if os.path.exists(filename):
+ content = open(filename, 'rb').read()
+ else:
+ content = requests.get(url).content
+ open(filename, 'wb').write(content)
+
+ return content.decode(errors='replace')
+
+def get_text(url):
+ # assume UTF-8
+
+ text = get_url(url)
+ if url in censor_urls:
+ text = censor_text(text)
+
+ heading = url.rsplit('/', 1)[-1]
+ return {
+ 'url': url,
+ 'text': text,
+ 'heading': heading,
+ 'length': len(text),
+ }
+
+def parse_span(line):
+ m = re_span_pointer.match(line)
+ if not m:
+ return None
+ return (m.group(1), int(m.group(2)), int(m.group(3)))
+
+def parse_edl(edl_text):
+ edl = {
+ 'spans': [],
+ 'links': [],
+ }
+ for line in edl_text.splitlines():
+ line = re_comment.sub('', line).strip()
+ if not line:
+ continue
+ span_pointer = parse_span(line)
+ if span_pointer:
+ edl['spans'].append(span_pointer)
+ continue
+ m = re_xanalink.match(line)
+ if m:
+ link_url = m.group(1)
+ edl['links'].append({
+ 'url': link_url,
+ 'text': get_url(link_url),
+ })
+ continue
+
+ return edl
+
+def get_span(text, url, start, length):
+ return {
+ 'url': url,
+ 'start': start,
+ 'length': length,
+ 'text': text[url][start:start + length]
+ }
+
+def fulfil_edl(edl):
+ text = {}
+ for url, start, length in parse_edl(edl):
+ if url not in text:
+ text[url] = get_text(url)
+
+ yield get_span(text, url, start, length)
+
+def get_urls(spans):
+ return {i[0] for i in spans}
+
+def url_filename(url):
+ return re_colon_slash.sub('_', url)
+
+def find_min_max(spans, source):
+ text_min, text_max = {}, {}
+ for url, start, length in spans:
+ if url in text_min:
+ text_min[url] = min(text_min[url], start)
+ else:
+ text_min[url] = start
+
+ if url in text_max:
+ text_max[url] = max(text_max[url], start + length)
+ else:
+ text_max[url] = start + length
+
+ for s in source:
+ url = s['url']
+ s['min'] = text_min[url]
+ s['max'] = text_max[url]
+
+def span_html(span_type, num):
+ return '<span class="{span_type} sourcedoc{span_type}" id="{span_type}{num}">'.format(num=num, span_type=span_type)
+
+def xanadoc_span_html(num, text, url, start, length, highlight=True, censor=False):
+ cls = []
+ if highlight:
+ cls = ['xanadoctransclusion', 'transclusion']
+ html_class = ' class="{}"'.format(' '.join(cls)) if cls else ''
+
+ html = '<span id="span{}"{} data-url="{}" data-start="{}" data-length="{}">{}</span>'.format(num, html_class, escape(url), start, length, text)
+
+ if censor:
+ return '<span class="censor">' + html + '</span>'
+ else:
+ return html
+
+def parse_sourcedoc_facet(facet):
+ leg = facet[0]
+ prefix = 'sourcedoc: '
+ assert leg.startswith(prefix)
+ return leg[len(prefix):]
+
+def parse_link(link_text):
+ link_type = None
+ expect = 'link_type'
+ facets = []
+ for line in link_text.splitlines():
+ line = re_comment.sub('', line).strip()
+ if not line:
+ continue
+ if expect == 'link_type':
+ if line.startswith('type='):
+ link_type = line[5:]
+ expect = 'facets'
+ continue
+ if expect != 'facets':
+ # print("unrecognized:", line)
+ continue
+
+ m = re_facet.match(line)
+ if m:
+ legs = []
+ facets.append(legs)
+ if m.group(1):
+ line = m.group(1)
+ else:
+ continue
+ if legs and legs[-1] == 'span:' and line.startswith('http'):
+ legs[-1] += ' ' + line
+ else:
+ legs.append(line.strip())
+ return {'type': link_type, 'facets': facets}
+
+def fulfil_edl_with_sources(edl_text):
+ edl = parse_edl(edl_text)
+ spans = edl['spans']
+
+ hide_transclusions = set()
+
+ two_facet_links = []
+
+ link_num = 0
+ for link in edl['links']:
+ link_detail = parse_link(link['text'])
+ if link_detail['type'] == 'HideTransclusions':
+ hide_transclusions.add(parse_sourcedoc_facet(link_detail['facets'][0]))
+ elif len(link_detail['facets']) == 2:
+ two_facet_links.append((link_num, [parse_span(span[0]) for span in link_detail['facets']]))
+ link_num += 1
+
+ source = [get_text(url) for url in get_urls(spans)]
+
+ source_text = {s['url']: s['text'] for s in source}
+
+ source_doc_links = defaultdict(list)
+
+ for link_num, facets in two_facet_links:
+ for span in facets:
+ url, start, length = span
+ source_doc_links[url].append((start, length, link_num, 'link'))
+ if url in source_text:
+ continue
+
+ s = get_text(url)
+ source.append(s)
+ source_text[s['url']] = s['text']
+
+ for s in source_doc_links.values():
+ s.sort()
+
+ spans = list(enumerate(spans))
+
+ doc_spans = []
+ for num, (url, start, length) in spans:
+ highlight = url not in hide_transclusions
+ span_text = source_text[url] # [start:start + length]
+ censor = url in censor_urls
+ new_text = ''
+ pos = start
+ for link_start, link_len, link_num, span_type in source_doc_links[url]:
+ link_end = link_start + link_len
+ if link_start >= start + length:
+ break
+ if link_end < start:
+ continue
+ open_tag = '<span class="xanadoclink link" id="xanalink{}">'.format(link_num)
+ link_span = (open_tag +
+ escape(span_text[link_start:link_end]) +
+ '</span>')
+ new_text += escape(span_text[pos:link_start]) + link_span
+ pos = link_end
+ new_text += escape(span_text[pos:start + length])
+ cur = xanadoc_span_html(num, new_text, url, start, length, highlight=highlight, censor=censor)
+ doc_spans.append(cur)
+
+ doc = ''.join(doc_spans)
+
+ for s in source:
+ text = s.pop('text')
+ if s['length'] > max_sourcedoc_size:
+ # print('{} > {}'.format(s['length'], max_sourcedoc_size))
+ continue
+ if s['url'] in hide_transclusions:
+ continue
+ source_spans = [(start, length, num, 'transclusion') for num, (url, start, length) in spans if url == s['url']]
+ source_spans += source_doc_links[s['url']]
+ source_spans.sort()
+
+ new_text = ''
+ pos = 0
+
+ for start, length, num, span_type in source_spans:
+ end = start + length
+ new_text += (escape(text[pos:start]) +
+ span_html(span_type, num) +
+ escape(text[start:end]) +
+ '</span>')
+ pos = end
+ new_text += escape(text[pos:])
+ new_text = new_text.replace('\n', '<br/>\n')
+
+ s['text'] = new_text
+
+ return {
+ 'source': source,
+ 'doc': doc.replace('\n', '<br/>\n'),
+ 'span_count': len(spans),
+ 'link_count': len(two_facet_links),
+ }
diff --git a/sourcing/static/bootstrap b/sourcing/static/bootstrap
new file mode 120000
index 0000000..fe0f86b
--- /dev/null
+++ b/sourcing/static/bootstrap
@@ -0,0 +1 @@
+/usr/share/javascript/bootstrap
\ No newline at end of file
diff --git a/sourcing/static/handlebars b/sourcing/static/handlebars
new file mode 120000
index 0000000..2a678c5
--- /dev/null
+++ b/sourcing/static/handlebars
@@ -0,0 +1 @@
+/usr/share/javascript/handlebars
\ No newline at end of file
diff --git a/sourcing/static/jquery b/sourcing/static/jquery
new file mode 120000
index 0000000..7e2f826
--- /dev/null
+++ b/sourcing/static/jquery
@@ -0,0 +1 @@
+/usr/share/javascript/jquery
\ No newline at end of file
diff --git a/sourcing/static/jquery-ui b/sourcing/static/jquery-ui
new file mode 120000
index 0000000..d9f803c
--- /dev/null
+++ b/sourcing/static/jquery-ui
@@ -0,0 +1 @@
+/usr/share/javascript/jquery-ui
\ No newline at end of file
diff --git a/sourcing/static/js/doc.js b/sourcing/static/js/doc.js
new file mode 100644
index 0000000..c0a90a2
--- /dev/null
+++ b/sourcing/static/js/doc.js
@@ -0,0 +1,15 @@
+$(function() {
+ $("div#right").hide();
+ /* $("button#add-new-span").click(function() {
+ $("div#right").show();
+ }); */
+ $("a.source-doc-link").click(function(e) {
+ var link = $(e.target);
+ var title = link.text();
+ console.log("source doc link " + title);
+ $("div#source-doc-heading").text("Add span from " + title);
+ $("#newSpanModal").modal('hide');
+ $("div#right").show();
+ $("div#source-doc-body").load(link.data("url"));
+ });
+});
diff --git a/sourcing/static/js/sourcedoc.js b/sourcing/static/js/sourcedoc.js
new file mode 100644
index 0000000..265cc31
--- /dev/null
+++ b/sourcing/static/js/sourcedoc.js
@@ -0,0 +1,49 @@
+function add_message() {
+ $("div#messages").text("whatever");
+}
+
+function show_selection(event) {
+ var range = window.getSelection().getRangeAt(0);
+ var start_element = range.startContainer.parentElement;
+ var end_element = range.endContainer.parentElement;
+
+ if ($(start_element).closest("div#right").prop("tagName") == 'DIV') {
+ return;
+ }
+
+ if (!start_element.hasAttribute('data-start') ||
+ !end_element.hasAttribute('data-start')) {
+ $("div#right").hide();
+ return;
+ }
+
+ var start = parseInt(start_element.getAttribute('data-start')) + range.startOffset;
+ var end = parseInt(end_element.getAttribute('data-start')) + range.endOffset;
+ var length = end - start;
+ if(length === 0) {
+ $("div#right").hide();
+ return;
+ }
+
+ $("#span").text(doc_url + ",start=" + start + ",length=" + length);
+
+ /*
+ $("div#right").show();
+ $("span#length").text(length);
+ $("input[name='offset']").val(start);
+ $("input[name='length']").val(length);
+ */
+}
+
+
+$(function() {
+ $("button#go").click(add_message);
+ $("div#right").hide();
+
+ $("body").mouseup(show_selection);
+
+ $("#show-span-selector").click(function(e) {
+ $("#span-selector").removeClass("hidden");
+ $(this).hide();
+ });
+});
diff --git a/sourcing/templates/edit.html b/sourcing/templates/edit.html
new file mode 100644
index 0000000..9268075
--- /dev/null
+++ b/sourcing/templates/edit.html
@@ -0,0 +1,17 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="edit " + doc.title %}
+
+{% set action=doc.edit_url %}
+{% set label="save" %}
+
+{% set fields %}
+{#
+{{ render_field(form.filename) }}
+{{ render_field(form.db_price_per_character) }}
+{{ render_field(form.db_document_price) }}
+#}
+{{ render_field(form.text, rows=20) }}
+{% endset %}
+
+{% include "form/simple.html" %}
diff --git a/sourcing/templates/flash_msg.html b/sourcing/templates/flash_msg.html
new file mode 100644
index 0000000..a390a28
--- /dev/null
+++ b/sourcing/templates/flash_msg.html
@@ -0,0 +1,10 @@
+{% with messages = get_flashed_messages() %}
+ {% if messages %}
+ {% for message in messages %}
+<div class="alert alert-success alert-dismissible" role="alert">
+ <button type="button" class="close" data-dismiss="alert"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
+ {{ message }}
+</div>
+ {% endfor %}
+ {% endif %}
+{% endwith %}
diff --git a/sourcing/templates/foot.html b/sourcing/templates/foot.html
new file mode 100644
index 0000000..6d6036f
--- /dev/null
+++ b/sourcing/templates/foot.html
@@ -0,0 +1,11 @@
+<hr/>
+<footer>© 2017 Project Xanadu</footer>
+</div>
+
+<script src="{{ url_for('static', filename='jquery/jquery.js') }}"></script>
+<script src="{{ url_for('static', filename='bootstrap/js/bootstrap.js') }}"></script>
+
+{{ scripts | safe }}
+
+</body>
+</html>
diff --git a/sourcing/templates/form/controls.html b/sourcing/templates/form/controls.html
new file mode 100644
index 0000000..1c00ff0
--- /dev/null
+++ b/sourcing/templates/form/controls.html
@@ -0,0 +1,33 @@
+{% macro render_field(field) %}
+<div class="form-group">
+ {{ field.label() }}
+ {{ field(class="form-control", **kwargs) }}
+ {% if field.description %}
+ <span class="help">{{ field.description|safe }}</span>
+ {% endif %}
+</div>
+{% endmacro %}
+
+{% macro checkbox(field) %}
+<div class="form-group">
+ <label class="checkbox-inline">
+ {{ field() }}
+ {{ field.label.text }}
+ </label>
+ {% if field.description %}
+ <span class="help">{{ field.description|safe }}</span>
+ {% endif %}
+</div>
+{% endmacro %}
+
+{% macro radio(field) %}
+<div class="form-group">
+ <label class="radio-inline">
+ {{ field() }}
+ {{ field.label.text }}
+ </label>
+ {% if field.description %}
+ <span class="help">{{ field.description|safe }}</span>
+ {% endif %}
+</div>
+{% endmacro %}
diff --git a/sourcing/templates/form/errors.html b/sourcing/templates/form/errors.html
new file mode 100644
index 0000000..e7ae1c7
--- /dev/null
+++ b/sourcing/templates/form/errors.html
@@ -0,0 +1,9 @@
+{% if form.errors %}
+<ul class="errors">
+ {% for field in form %}
+ {% for error in form.errors[field.name] %}
+ <li>{{ field.label }}: {{ error }}</li>
+ {% endfor %}
+ {% endfor %}
+</ul>
+{% endif %}
diff --git a/sourcing/templates/form/main.html b/sourcing/templates/form/main.html
new file mode 100644
index 0000000..a57b6fa
--- /dev/null
+++ b/sourcing/templates/form/main.html
@@ -0,0 +1,11 @@
+<div class="well">
+ {% include "form/errors.html" %}
+ <form action="{{action}}" method="{{method|default('post')}}" role="form">
+ {{ form.hidden_tag() }}
+ {{ fields | safe }}
+ <div class="form-group">
+ <button type="submit" class="btn btn-default">{{ label }}</button>
+ </div>
+ </form>
+</div>
+
diff --git a/sourcing/templates/form/simple.html b/sourcing/templates/form/simple.html
new file mode 100644
index 0000000..97a9a68
--- /dev/null
+++ b/sourcing/templates/form/simple.html
@@ -0,0 +1,10 @@
+{% include "head.html" %}
+
+<div class="row">
+ <div class="col-md-12">
+ <h1>{{ title }}</h1>
+ {% include "form/main.html" %}
+ </div>
+</div>
+
+{% include "foot.html" %}
diff --git a/sourcing/templates/head.html b/sourcing/templates/head.html
new file mode 100644
index 0000000..3ff6f28
--- /dev/null
+++ b/sourcing/templates/head.html
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{ title | default("Xanadu") }}</title>
+
+<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap/css/bootstrap.css') }}">
+</head>
+
+<body>
+{% include "navbar.html" %}
+{{ dialogs | safe }}
+<div class="container">
+{% include "flash_msg.html" %}
diff --git a/sourcing/templates/home.html b/sourcing/templates/home.html
new file mode 100644
index 0000000..84f6952
--- /dev/null
+++ b/sourcing/templates/home.html
@@ -0,0 +1,23 @@
+{% include "head.html" %}
+<h1>perma.pub</h1>
+
+<ul>
+{% for doc in docs %}
+<li>
+<a href="{{ doc.url }}">{{ doc.title() }}</a>
+by {{ doc.user.username }}
+— created {{ doc.created | datetime }}
+</li>
+{% endfor %}
+</ul>
+
+<p>
+<a href="{{ url_for('.new_sourcedoc') }}" class="btn btn-default">new source document</a>
+<a href="{{ url_for('.new_xanalink') }}" class="btn btn-default">new xanalink</a>
+<a href="{{ url_for('.new_xanadoc') }}" class="btn btn-default">new xanadoc</a>
+{#
+<a href="#" class="btn btn-default">upload a source document</a>
+#}
+</p>
+
+{% include "foot.html" %}
diff --git a/sourcing/templates/login.html b/sourcing/templates/login.html
new file mode 100644
index 0000000..3b97ea7
--- /dev/null
+++ b/sourcing/templates/login.html
@@ -0,0 +1,13 @@
+{% from "form/controls.html" import render_field, checkbox, submit %}
+
+{% set title="Login" %}
+{% set action=url_for('.login') %}
+{% set label="Sign in" %}
+
+{% set fields %}
+{{ render_field(form.user_or_email) }}
+{{ render_field(form.password) }}
+{{ checkbox(form.remember) }}
+{% endset %}
+
+{% include "form/simple.html" %}
diff --git a/sourcing/templates/mail/signup.txt b/sourcing/templates/mail/signup.txt
new file mode 100644
index 0000000..ee3f593
--- /dev/null
+++ b/sourcing/templates/mail/signup.txt
@@ -0,0 +1 @@
+You signed up for xanadu
diff --git a/sourcing/templates/navbar.html b/sourcing/templates/navbar.html
new file mode 100644
index 0000000..24bf377
--- /dev/null
+++ b/sourcing/templates/navbar.html
@@ -0,0 +1,34 @@
+<nav class="navbar navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="{{ url_for('.home') }}">{{ config.SITE_NAME }}</a>
+ </div>
+
+ <div id="navbar" class="navbar-collapse collapse">
+ <ul class="nav navbar-nav">
+ {% for page in ('home', 'about', 'contact') %}
+ <li{% if current_page == page %} class="active"{% endif %}><a href="{{ url_for('.' + page) }}">{{ page | title }}</a></li>
+ {% endfor %}
+ </ul>
+ <ul class="nav navbar-nav navbar-right">
+ {% if current_user.is_authenticated %}
+ {% if current_user.admin %}
+ <li><a href="{{ url_for('.admin') }}">admin</a></li>
+ {% endif %}
+ <li><a href="{{ url_for('.account_settings') }}">{{ current_user.username }}</a></li>
+ <li><a href="{{ url_for('.logout') }}">logout</a></li>
+ {% else %}
+ <li><a href="{{ url_for('.signup') }}">signup</a></li>
+ <li><a href="{{ url_for('.login') }}">login</a></li>
+ {% endif %}
+ </ul>
+ </div>
+ </div>
+</nav>
+
diff --git a/sourcing/templates/new.html b/sourcing/templates/new.html
new file mode 100644
index 0000000..8d8e943
--- /dev/null
+++ b/sourcing/templates/new.html
@@ -0,0 +1,15 @@
+{% from "form/controls.html" import render_field %}
+
+{% set action=url_for(request.endpoint) %}
+{% set label="save" %}
+
+{% set fields %}
+{#
+{{ render_field(form.filename) }}
+{{ render_field(form.db_price_per_character) }}
+{{ render_field(form.db_document_price) }}
+#}
+{{ render_field(form.text, rows=20) }}
+{% endset %}
+
+{% include "form/simple.html" %}
diff --git a/sourcing/templates/signup.html b/sourcing/templates/signup.html
new file mode 100644
index 0000000..4d9448f
--- /dev/null
+++ b/sourcing/templates/signup.html
@@ -0,0 +1,13 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="Sign up" %}
+{% set action=url_for('.signup') %}
+{% set label="Sign up" %}
+
+{% set fields %}
+{{ render_field(form.username) }}
+{{ render_field(form.password) }}
+{{ render_field(form.email) }}
+{% endset %}
+
+{% include "form/simple.html" %}
diff --git a/sourcing/templates/source_doc.html b/sourcing/templates/source_doc.html
new file mode 100644
index 0000000..ebfe45a
--- /dev/null
+++ b/sourcing/templates/source_doc.html
@@ -0,0 +1,33 @@
+{% set title = doc.filename %}
+{% set price = doc.document_price %}
+{% include "head.html" %}
+
+<style>
+div#text { font-family: Courier; }
+</style>
+
+<div class="row">
+ <div class="col-xs-12">
+ <h1>{{ title }}
+ {% if doc.user == current_user %}
+ <a href="{{ url_for('.edit_source_document',
+ filename=doc.filename) }}" class="btn btn-default">edit</a>
+ {% endif %}
+ </h1>
+ <p><a href="{{ url_for('.home') }}">back to index</a></p>
+ <ul>
+ <li>character count: {{ '{:,d}'.format(doc.text | length) }}</li>
+ {% if price %}
+ <li>document price: {{ '{:,.2f}'.format(price) }} nanobucks</li>
+ <li>price per character: {{ '{:,.2f}'.format(doc.price_per_character) }} nanobucks</li>
+ {% endif %}
+ </ul>
+ <div class="well" id="text">
+ {%- for start, line in iter_lines(doc.get_text()) if line -%}
+ <p data-start="{{ start }}">{{ line }}</p>
+ {%- endfor -%}
+ </div>
+ </div>
+</div>
+
+{% include "foot.html" %}
diff --git a/sourcing/templates/user/account.html b/sourcing/templates/user/account.html
new file mode 100644
index 0000000..ad165ac
--- /dev/null
+++ b/sourcing/templates/user/account.html
@@ -0,0 +1,12 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="account settings" %}
+
+{% set action=url_for(".account_settings") %}
+{% set label="save" %}
+
+{% set fields %}
+{{ render_field(form.full_name) }}
+{% endset %}
+
+{% include "form/simple.html" %}
diff --git a/sourcing/templates/view.html b/sourcing/templates/view.html
new file mode 100644
index 0000000..628bc2d
--- /dev/null
+++ b/sourcing/templates/view.html
@@ -0,0 +1,34 @@
+{% set title = doc.title() %}
+{% include "head.html" %}
+
+<style>
+div#text { font-family: Courier; }
+</style>
+
+<div class="row">
+ <div class="col-xs-12">
+ <h1>{{ title }}
+ {% if doc.user == current_user %}
+ <a href="{{ doc.edit_url }}" class="btn btn-default">edit</a>
+ {% endif %}
+ </h1>
+ <p><a href="{{ url_for('.home') }}">back to index</a></p>
+ <div class="well" id="text">
+ {%- for start, line in iter_lines(doc.text) if line -%}
+ <p data-start="{{ start }}">{{ line }}</p>
+ {%- endfor -%}
+ </div>
+ {% if doc.type == 'sourcedoc' %}
+ <a href="#" id="show-span-selector" class="btn btn-default">show span selector</a>
+ <p id="span-selector" class="hidden">span: <span id="span"></span></p>
+ {% set scripts %}
+ <script>
+ var doc_url = '{{ doc.external_url }}';
+ </script>
+ <script src="{{ url_for('static', filename='js/sourcedoc.js') }}"></script>
+ {% endset %}
+ {% endif %}
+ </div>
+</div>
+
+{% include "foot.html" %}
diff --git a/sourcing/text.py b/sourcing/text.py
new file mode 100644
index 0000000..cbfea91
--- /dev/null
+++ b/sourcing/text.py
@@ -0,0 +1,21 @@
+import re
+import random
+
+re_newline = re.compile('\r?\n')
+
+def find_newlines(text):
+ return (m.end(0) for m in re_newline.finditer(text))
+
+def iter_lines(text):
+ start = 0
+ for m in re_newline.finditer(text):
+ end = m.end(0)
+ yield (start, text[start:end])
+ start = m.end(0)
+ if start < len(text) - 1:
+ yield (start, text[start:])
+
+def censor_text(text):
+ def random_chr():
+ return chr(random.randint(9728, 9983))
+ return ''.join(random_chr() if c.isalnum() else c for c in text)
diff --git a/sourcing/utils.py b/sourcing/utils.py
new file mode 100644
index 0000000..701e875
--- /dev/null
+++ b/sourcing/utils.py
@@ -0,0 +1,13 @@
+import humanize
+from datetime import date, timedelta
+
+def display_datetime(dt):
+ if dt is None:
+ return 'n/a'
+ today = date.today()
+ if today - dt.date() < timedelta(days=1):
+ return humanize.naturaltime(dt)
+ else:
+ return dt.strftime('%a, %d %b %Y')
+
+
diff --git a/sourcing/view.py b/sourcing/view.py
new file mode 100644
index 0000000..4029456
--- /dev/null
+++ b/sourcing/view.py
@@ -0,0 +1,232 @@
+from flask import (Blueprint, render_template, request, redirect, flash,
+ url_for, abort, jsonify)
+from flask_login import (login_user, current_user, logout_user,
+ login_required, LoginManager)
+from .forms import (LoginForm, SignupForm, AccountSettingsForm,
+ UploadSourceDocForm, SourceDocForm, ItemForm)
+from .model import User, SourceDoc, Item, XanaDoc, XanaLink
+from .database import session
+from .text import iter_lines
+from werkzeug.debug.tbtools import get_current_traceback
+from jinja2 import evalcontextfilter, Markup
+from functools import wraps
+
+import re
+
+login_manager = LoginManager()
+login_manager.login_view = 'login'
+re_paragraph = re.compile(r'(?:\r\n|\r|\n){2,}')
+bp = Blueprint('view', __name__)
+
+def init_app(app):
+ login_manager.init_app(app)
+ app.register_blueprint(bp)
+
+ @app.template_filter()
+ @evalcontextfilter
+ def newline_html(eval_ctx, value):
+ return u'\n\n'.join(Markup(u'<p>') + p.replace('\n', Markup('<br>')) + Markup(u'</p>')
+ for p in re_paragraph.split(value))
+
+@login_manager.user_loader
+def load_user(user_id):
+ return User.query.get(user_id)
+
+
+# where do we redirect after signup is complete
+view_after_signup = '.home'
+
+@bp.context_processor
+def inject_user():
+ return dict(current_user=current_user)
+
+def show_errors(f):
+ @wraps(f)
+ def wrapper(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except Exception:
+ traceback = get_current_traceback(skip=1, show_hidden_frames=False,
+ ignore_system_exceptions=True)
+ return traceback.render_full().encode('utf-8', 'replace')
+ return wrapper
+
+@bp.route('/')
+def home():
+ titles = XanaLink.get_all_titles()
+ docs = Item.query.order_by(Item.created)
+ return render_template('home.html', docs=docs)
+
+@bp.route('/source_doc_upload', methods=["POST"])
+@show_errors
+def source_doc_upload():
+ f = request.files['sourcedoc_file']
+ text = f.read()
+ doc = SourceDoc(text=text, user=current_user, filename=f.filename)
+ session.add(doc)
+ session.commit()
+ flash('new source document uploaded')
+ return redirect(doc.url)
+
+@bp.route('/about')
+def about():
+ return render_template('about.html')
+
+@bp.route('/contact')
+def contact():
+ return render_template('contact.html')
+
+def redirect_to_home():
+ return redirect(url_for('.home'))
+
+@bp.route('/login', methods=['GET', 'POST'])
+def login():
+ form = LoginForm(next=request.args.get('next'))
+ if form.validate_on_submit():
+ login_user(form.user, remember=form.remember.data)
+ flash('Logged in successfully.')
+ return redirect(request.form.get('next') or url_for('.home'))
+ return render_template('login.html', form=form)
+
+@bp.route('/logout')
+def logout():
+ logout_user()
+ flash('You have been logged out.')
+ return redirect_to_home()
+
+@bp.route('/signup', methods=['GET', 'POST'])
+def signup():
+ form = SignupForm()
+ if not form.validate_on_submit():
+ return render_template('signup.html', form=form)
+
+ user = User(**form.data)
+ session.add(user)
+ session.commit()
+ flash('New account created.')
+ login_user(user)
+ return redirect(url_for(view_after_signup))
+
+def redirect_to_doc(doc):
+ return redirect(url_for('.view_document', hashid=doc.hashid))
+
+def get_source_doc(username, hashid):
+ doc = Item.get_by_hashid(hashid)
+ if doc and doc.user.username != username:
+ doc = None
+ return doc if doc else abort(404)
+
+def get_item(username, hashid):
+ doc = Item.get_by_hashid(hashid)
+ if doc and doc.user.username != username:
+ doc = None
+ return doc if doc else abort(404)
+
+@bp.route('/<username>/<hashid>')
+def view_item(username, hashid):
+ return render_template('view.html',
+ doc=get_item(username, hashid),
+ iter_lines=iter_lines)
+
+@bp.route('/<username>/<hashid>/edit', methods=['GET', 'POST'])
+def edit_item(username, hashid):
+ obj = get_item(current_user.username, hashid)
+ form = SourceDocForm(obj=obj)
+ if form.validate_on_submit():
+ form.populate_obj(obj)
+ session.add(obj)
+ session.commit()
+ flash('Changes to {} saved.'.format(obj.type))
+ return redirect(obj.url)
+ return render_template('edit.html', form=form, doc=obj)
+
+@bp.route('/source_doc_text/<int:source_doc_id>')
+def source_doc_text(source_doc_id):
+ doc = SourceDoc.query.get(source_doc_id)
+
+ return render_template('source_doc_text.html', doc=doc, iter_lines=iter_lines)
+
+@bp.route('/settings/account', methods=['GET', 'POST'])
+@login_required
+def account_settings():
+ form = AccountSettingsForm(obj=current_user)
+ if form.validate_on_submit():
+ form.populate_obj(current_user)
+ session.add(current_user)
+ session.commit()
+ flash('Account details updated.')
+ return redirect(url_for(request.endpoint))
+ return render_template('user/account.html', form=form)
+
+@bp.route('/new/sourcedoc', methods=['GET', 'POST'])
+@login_required
+def new_sourcedoc():
+ form = SourceDocForm()
+ if form.validate_on_submit():
+ doc = SourceDoc(user=current_user)
+ form.populate_obj(doc)
+ session.add(doc)
+ session.commit()
+ flash('New document saved.')
+ return redirect(doc.url)
+ return render_template('new.html', form=form, title='source document')
+
+@bp.route('/new/xanalink', methods=['GET', 'POST'])
+@login_required
+def new_xanalink():
+ form = ItemForm()
+ if form.validate_on_submit():
+ obj = XanaLink(user=current_user)
+ form.populate_obj(obj)
+ session.add(obj)
+ session.commit()
+ flash('New xanalink saved.')
+ return redirect(obj.url)
+ return render_template('new.html', form=form, title='xanalink')
+
+@bp.route('/new/xanadoc', methods=['GET', 'POST'])
+@login_required
+def new_xanadoc():
+ form = ItemForm()
+ if form.validate_on_submit():
+ obj = XanaLink(user=current_user)
+ form.populate_obj(obj)
+ session.add(obj)
+ session.commit()
+ flash('New xanalink saved.')
+ return redirect(obj.url)
+ return render_template('new.html', form=form, title='xanadoc')
+
+@bp.route('/edit/<filename>', methods=['GET', 'POST'])
+@login_required
+def edit_source_document(filename):
+ doc = get_source_doc(current_user.username, filename)
+ form = SourceDocForm(obj=doc)
+ if form.validate_on_submit():
+ form.populate_obj(doc)
+ session.add(doc)
+ session.commit()
+ flash('Changes to document saved.')
+ return redirect(doc.url)
+ return render_template('edit.html', form=form, doc=doc)
+
+@bp.route('/api/1/get/<username>/<filename>')
+def api_get_document(username, filename):
+ doc = get_source_doc(username, filename)
+ if not doc:
+ return abort(404)
+ ret = {
+ 'username': username,
+ 'filename': filename,
+ 'character_count': len(doc.text),
+ 'document_price': str(doc.document_price),
+ 'price_per_character': str(doc.price_per_character),
+ }
+ return jsonify(ret)
+
+@bp.route('/all_titles')
+def get_all_titles():
+ titles = XanaLink.get_all_titles()
+ for k, v in titles.items():
+ print(from_external(k), v)
+ return ''
diff --git a/titles.py b/titles.py
new file mode 100755
index 0000000..311843a
--- /dev/null
+++ b/titles.py
@@ -0,0 +1,10 @@
+#!/usr/bin/python3
+
+from sourcing import create_app
+from sourcing.model import XanaLink, from_external
+
+if __name__ == "__main__":
+ app = create_app('config.default')
+
+ with app.test_client() as c:
+ c.get('/all_titles')