initial commit

autoapp.py

@@ -0,0 +1,2 @@
+from sourcing import create_app
+app = create_app('config.default')

create_db.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+
+from sourcing.model import Base
+from sourcing.database import init_db, session
+from sourcing import create_app
+
+app = create_app('config.default')
+db_url = app.config['DB_URL']
+init_db(db_url)
+engine = session.get_bind()
+# Base.metadata.drop_all(engine)
+Base.metadata.create_all(engine)

run.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python3
+
+from sourcing import create_app
+
+if __name__ == "__main__":
+    app = create_app('config.default')
+    app.run('0.0.0.0', debug=True)

sourcing/__init__.py

@@ -0,0 +1,12 @@
+from flask import Flask
+from . import database
+from . import view
+from .utils import display_datetime
+
+def create_app(config):
+    app = Flask(__name__)
+    app.config.from_object(config)
+    database.init_app(app)
+    view.init_app(app)
+    app.jinja_env.filters['datetime'] = display_datetime
+    return app

sourcing/database.py

@@ -0,0 +1,18 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+session = scoped_session(sessionmaker())
+
+def init_db(db_url):
+    session.configure(bind=get_engine(db_url))
+
+def get_engine(db_url):
+    return create_engine(db_url, pool_recycle=3600)
+
+def init_app(app):
+    db_url = app.config['DB_URL']
+    session.configure(bind=get_engine(db_url))
+
+    @app.teardown_appcontext
+    def shutdown_session(exception=None):
+        session.remove()

sourcing/forms.py

@@ -0,0 +1,75 @@
+from flask_wtf import Form
+from wtforms.fields import StringField, PasswordField, BooleanField, HiddenField, TextAreaField, RadioField, FileField, IntegerField
+from wtforms.validators import InputRequired, Email, Length, ValidationError, Regexp, NoneOf, Optional
+from .model import User, LoginError, re_username, reserved_name, user_exists
+
+PASSWORD_LEN = 64
+EMAIL_LEN = 64
+
+class SignupForm(Form):
+    username = StringField('username',
+                           [InputRequired(),
+                            Regexp(re_username),
+                            NoneOf(reserved_name, message='Not available.'),
+                            Length(min=3, max=64)],
+                           [lambda name: name and name.replace(' ', '_')])
+    email = StringField('e-mail address',
+                        [InputRequired(), Email(),
+                         Length(min=5, max=EMAIL_LEN)],
+                        description="we never share your e-mail address")
+    password = PasswordField('password',
+                        [InputRequired(), Length(min=4, max=PASSWORD_LEN)])
+
+    def validate_username(form, field):
+        if user_exists(User.username, field.data):
+            raise ValidationError('Not available')
+
+    def validate_email(form, field):
+        if user_exists(User.email, field.data):
+            raise ValidationError('In use by another account')
+
+class LoginForm(Form):
+    user_or_email = StringField('username or e-mail address',
+                                [InputRequired(), Length(min=3, max=EMAIL_LEN)],
+                                [lambda name: name and name.replace(' ', '_')])
+    password = PasswordField('password',
+                             [InputRequired(), Length(max=PASSWORD_LEN)])
+    remember = BooleanField('stay logged in')
+    next = HiddenField('next')
+
+    def validate(self):
+        rv = Form.validate(self)
+        if not rv:
+            return False
+
+        try:
+            self.user = User.attempt_login(self.user_or_email.data,
+                                           self.password.data)
+            return True
+        except LoginError as e:
+            self.user_or_email.errors.append(e.msg)
+            return False
+
+class ForgotPasswordForm(Form):
+    username_or_email = StringField('username or e-mail address',
+                                    [InputRequired(), Length(max=EMAIL_LEN)])
+
+class AccountSettingsForm(Form):
+    full_name = StringField('full name', [Length(max=64)])
+
+class ChangePasswordForm(Form):
+    old_password = PasswordField('current password',
+                                 [InputRequired(), Length(max=PASSWORD_LEN)])
+    new_password = PasswordField('new password',
+                                 [InputRequired(), Length(max=PASSWORD_LEN)])
+
+class SourceDocForm(Form):
+    text = TextAreaField('text', [InputRequired()])
+    db_price_per_character = IntegerField('price per character', [Optional()])
+    db_document_price = IntegerField('document price', [Optional()])
+
+class ItemForm(Form):
+    text = TextAreaField('text', [InputRequired()])
+
+class UploadSourceDocForm(Form):
+    sourcedoc_file = FileField('SourceDoc', [Regexp(r'^[^/\\]+\.txt$')])

sourcing/model.py

@@ -0,0 +1,234 @@
+from flask import url_for
+from .database import session
+from .parse import parse_link, parse_sourcedoc_facet, parse_span
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, ForeignKey
+from sqlalchemy.types import String, Unicode, Integer, DateTime, Boolean, UnicodeText, Enum
+from sqlalchemy import func
+from sqlalchemy.orm import relationship, validates, synonym
+from sqlalchemy.sql import exists
+from flask_login import UserMixin
+from werkzeug.security import generate_password_hash, check_password_hash
+from sqlalchemy.ext.hybrid import hybrid_property
+import re
+
+from hashids import Hashids
+
+doc_hashids = Hashids(min_length=8)
+
+Base = declarative_base()
+Base.query = session.query_property()
+
+# list of disallowed usernames - maybe this should be in the database
+reserved_name = ['root', 'admin', 'administrator', 'support', 'info',
+    'test', 'tech', 'online', 'old', 'new', 'jobs', 'login', 'job', 'ipad'
+    'iphone', 'javascript', 'script', 'host', 'mail', 'image', 'faq',
+    'file', 'ftp', 'error', 'warning', 'the', 'assistance', 'maintenance',
+    'controller', 'head', 'chief', 'anon']
+
+re_username = re.compile('^\w+$', re.U)
+re_full_name = re.compile('^([-.\'" ]|[^\W\d_])+$', re.U)
+
+def user_exists(field, value):
+    return session.query(exists().where(field == value)).scalar()
+
+class TimeStampedModel(Base):
+    __abstract__ = True
+    created = Column(DateTime, default=func.now())
+    modified = Column(DateTime, default=func.now(), onupdate=func.now())
+
+class LoginError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+
+class User(TimeStampedModel, UserMixin):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(Unicode(32), unique=True, nullable=False)
+    pw_hash = Column(String(160), nullable=False)
+    email = Column(Unicode(64), unique=True, nullable=False)
+    email_verified = Column(Boolean(), nullable=False, default=False)
+    disabled = Column(Boolean(), nullable=False, default=False)
+    deleted = Column(Boolean(), nullable=False, default=False)
+    is_super = Column(Boolean, nullable=False, default=False)
+    last_login = Column(DateTime)
+    full_name = Column(Unicode(64))
+    balance = Column(Integer, nullable=False, default=0)
+
+    user_id = synonym('id')
+    name = synonym('full_name')
+    user_name = synonym('username')
+
+    def __init__(self, **kwargs):
+        pw_hash = generate_password_hash(kwargs.pop('password'))
+        return super(User, self).__init__(pw_hash=pw_hash, **kwargs)
+
+    def __repr__(self):
+        return '<User: {!r}>'.format(self.username)
+
+    def set_password(self, password):
+        self.pw_hash = generate_password_hash(password)
+
+    def check_password(self, password):
+        return check_password_hash(self.pw_hash, password)
+
+    def get_id(self):
+        return self.id
+
+    @validates('email')
+    def validate_email(self, key, value):
+        assert '@' in value
+        return value
+
+    @validates('username')
+    def validate_usernane(self, key, value):
+        assert re_username.match(value)
+        return value
+
+    @validates('full_name')
+    def validate_full_name(self, key, value):
+        if value:
+            assert re_full_name.match(value)
+        return value
+
+    @hybrid_property
+    def is_live(self):
+        return self.email_verified & ~self.disabled & ~self.deleted
+
+    @classmethod
+    def lookup_user_or_email(cls, user_or_email):
+        field = cls.email if '@' in user_or_email else cls.username
+        return cls.query.filter(field == user_or_email).one_or_none()
+
+    @property
+    def mail_to_name(self):
+        '''Name to use on e-mails sent to the user.'''
+        return self.full_name or self.username
+
+    @classmethod
+    def attempt_login(cls, user_or_email, password):
+        user = cls.lookup_user_or_email(user_or_email)
+        if not user:
+            raise LoginError('user not found')
+        if user.disabled:
+            raise LoginError('user account disabled')
+        if not user.check_password(password):
+            raise LoginError('incorrect password')
+        return user
+
+class Item(TimeStampedModel):
+    __tablename__ = 'item'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    published = Column(DateTime)
+    type = Column(Enum('sourcedoc', 'xanadoc', 'xanalink', name='item_type'), nullable=False)
+    filename = Column(Unicode)
+    text = Column(UnicodeText)
+
+    user = relationship('User', backref='items')
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'item',
+        'polymorphic_on': type,
+    }
+
+    @property
+    def hashid(self):
+        return doc_hashids.encode(self.id)
+
+    @classmethod
+    def get_by_hashid(cls, hashid):
+        return cls.query.get(doc_hashids.decode(hashid))
+
+    @property
+    def url(self):
+        return url_for('.view_item', username=self.user.username, hashid=self.hashid)
+
+    @property
+    def external_url(self):
+        return url_for('.view_item',
+                       username=self.user.username,
+                       hashid=self.hashid,
+                       _external=True)
+
+    @property
+    def edit_url(self):
+        return url_for('.edit_item', username=self.user.username, hashid=self.hashid)
+
+    def title(self, titles=None):
+        if not titles:
+            titles = XanaLink.get_all_titles()
+        return self.type + ": " + titles.get(self, self.hashid)
+
+    @classmethod
+    def from_external(cls, url):
+        home = url_for('.home', _external=True)
+        if not url.startswith(home):
+            return
+        username, _, hashid = url[len(home):].partition('/')
+        q = cls.query.filter(User.username == username,
+                             cls.id == doc_hashids.decode(hashid)[0])
+        return q.one_or_none()
+
+class XanaDoc(Item):
+    __tablename__ = 'xanadoc'
+    id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+
+    __mapper_args__ = {'polymorphic_identity': 'xanadoc'}
+
+class XanaLink(Item):
+    __tablename__ = 'xanalink'
+    id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+
+    __mapper_args__ = {'polymorphic_identity': 'xanalink'}
+
+    def parse(self):
+        return parse_link(self.text)
+
+    def title(self, titles=None):
+        if titles is None:
+            titles = XanaLink.get_all_titles()
+        if self in titles:
+            return self.type + ": " + titles[self]
+
+        parsed = self.parse()
+
+        if parsed['type'] == 'title':
+            ident = parsed['facets'][0][0].partition(': ')[2]
+            item = Item.from_external(ident)
+            if item in titles:
+                return parsed['type'] + " link for " + item.title(titles=titles)
+
+        return parsed['type'] + " link: " + self.hashid
+
+    @classmethod
+    def get_all_titles(cls):
+        titles = {}
+        for link in (obj.parse() for obj in cls.query):
+            if link['type'] != 'title':
+                continue
+            facet1, facet2 = link['facets']
+            link_type, _, ident = facet1[0].partition(': ')
+            item = Item.from_external(ident)
+            print(ident, item)
+
+            ident2, start, length = parse_span(facet2[0])
+            title = SourceDoc.from_external(ident2).text[start:length + start]
+            titles[item] = title
+        return titles
+
+class SourceDoc(Item):
+    __tablename__ = 'sourcedoc'
+    id = Column(Integer, ForeignKey('item.id'), primary_key=True)
+    db_price_per_character = Column(Integer)
+    db_document_price = Column(Integer)
+
+    @property
+    def document_price(self):
+        return self.db_document_price or self.db_price_per_character * len(self.text)
+
+    @property
+    def price_per_character(self):
+        return self.db_price_per_character or self.db_document_price / len(self.text)
+
+    __mapper_args__ = {'polymorphic_identity': 'sourcedoc'}

sourcing/parse.py

@@ -0,0 +1,273 @@
+import re
+import requests
+import os.path
+import random
+from html import escape
+from collections import defaultdict
+
+re_comment = re.compile(r'#.*')
+re_span_pointer = re.compile(r'span: (.*),start=(\d+),length=(\d+)')
+re_xanalink = re.compile('xanalink: +([^ ]+) *$')
+re_facet = re.compile('^facet\d* *=\s*(.*)\s*$')
+
+re_colon_slash = re.compile('[/:]+')
+
+project_dir = os.path.dirname(os.path.dirname(__file__))
+cache_location = os.path.join(project_dir, 'cache')
+
+max_sourcedoc_size = 600000
+
+xnb_per_char = 150000
+
+censor_urls = {'http://hyperland.com/xuCambDemo/J.Ineffable.txt',
+               'http://royalty.pub/Ineffable/17/32/Trovato'}
+
+def censor_text(text):
+    return ''.join(chr(random.randint(9728, 9983)) if c.isalnum() else c
+                   for c in text)
+
+def get_url(url):
+    filename = os.path.join(cache_location, url_filename(url))
+
+    if os.path.exists(filename):
+        content = open(filename, 'rb').read()
+    else:
+        content = requests.get(url).content
+        open(filename, 'wb').write(content)
+
+    return content.decode(errors='replace')
+
+def get_text(url):
+    # assume UTF-8
+
+    text = get_url(url)
+    if url in censor_urls:
+        text = censor_text(text)
+
+    heading = url.rsplit('/', 1)[-1]
+    return {
+        'url': url,
+        'text': text,
+        'heading': heading,
+        'length': len(text),
+    }
+
+def parse_span(line):
+    m = re_span_pointer.match(line)
+    if not m:
+        return None
+    return (m.group(1), int(m.group(2)), int(m.group(3)))
+
+def parse_edl(edl_text):
+    edl = {
+        'spans': [],
+        'links': [],
+    }
+    for line in edl_text.splitlines():
+        line = re_comment.sub('', line).strip()
+        if not line:
+            continue
+        span_pointer = parse_span(line)
+        if span_pointer:
+            edl['spans'].append(span_pointer)
+            continue
+        m = re_xanalink.match(line)
+        if m:
+            link_url = m.group(1)
+            edl['links'].append({
+                'url': link_url,
+                'text': get_url(link_url),
+            })
+            continue
+
+    return edl
+
+def get_span(text, url, start, length):
+    return {
+        'url': url,
+        'start': start,
+        'length': length,
+        'text': text[url][start:start + length]
+    }
+
+def fulfil_edl(edl):
+    text = {}
+    for url, start, length in parse_edl(edl):
+        if url not in text:
+            text[url] = get_text(url)
+
+        yield get_span(text, url, start, length)
+
+def get_urls(spans):
+    return {i[0] for i in spans}
+
+def url_filename(url):
+    return re_colon_slash.sub('_', url)
+
+def find_min_max(spans, source):
+    text_min, text_max = {}, {}
+    for url, start, length in spans:
+        if url in text_min:
+            text_min[url] = min(text_min[url], start)
+        else:
+            text_min[url] = start
+
+        if url in text_max:
+            text_max[url] = max(text_max[url], start + length)
+        else:
+            text_max[url] = start + length
+
+    for s in source:
+        url = s['url']
+        s['min'] = text_min[url]
+        s['max'] = text_max[url]
+
+def span_html(span_type, num):
+    return '<span class="{span_type} sourcedoc{span_type}" id="{span_type}{num}">'.format(num=num, span_type=span_type)
+
+def xanadoc_span_html(num, text, url, start, length, highlight=True, censor=False):
+    cls = []
+    if highlight:
+        cls = ['xanadoctransclusion', 'transclusion']
+    html_class = ' class="{}"'.format(' '.join(cls)) if cls else ''
+
+    html = '<span id="span{}"{} data-url="{}" data-start="{}" data-length="{}">{}</span>'.format(num, html_class, escape(url), start, length, text)
+
+    if censor:
+        return '<span class="censor">' + html + '</span>'
+    else:
+        return html
+
+def parse_sourcedoc_facet(facet):
+    leg = facet[0]
+    prefix = 'sourcedoc: '
+    assert leg.startswith(prefix)
+    return leg[len(prefix):]
+
+def parse_link(link_text):
+    link_type = None
+    expect = 'link_type'
+    facets = []
+    for line in link_text.splitlines():
+        line = re_comment.sub('', line).strip()
+        if not line:
+            continue
+        if expect == 'link_type':
+            if line.startswith('type='):
+                link_type = line[5:]
+                expect = 'facets'
+            continue
+        if expect != 'facets':
+            # print("unrecognized:", line)
+            continue
+
+        m = re_facet.match(line)
+        if m:
+            legs = []
+            facets.append(legs)
+            if m.group(1):
+                line = m.group(1)
+            else:
+                continue
+        if legs and legs[-1] == 'span:' and line.startswith('http'):
+            legs[-1] += ' ' + line
+        else:
+            legs.append(line.strip())
+    return {'type': link_type, 'facets': facets}
+
+def fulfil_edl_with_sources(edl_text):
+    edl = parse_edl(edl_text)
+    spans = edl['spans']
+
+    hide_transclusions = set()
+
+    two_facet_links = []
+
+    link_num = 0
+    for link in edl['links']:
+        link_detail = parse_link(link['text'])
+        if link_detail['type'] == 'HideTransclusions':
+            hide_transclusions.add(parse_sourcedoc_facet(link_detail['facets'][0]))
+        elif len(link_detail['facets']) == 2:
+            two_facet_links.append((link_num, [parse_span(span[0]) for span in link_detail['facets']]))
+            link_num += 1
+
+    source = [get_text(url) for url in get_urls(spans)]
+
+    source_text = {s['url']: s['text'] for s in source}
+
+    source_doc_links = defaultdict(list)
+
+    for link_num, facets in two_facet_links:
+        for span in facets:
+            url, start, length = span
+            source_doc_links[url].append((start, length, link_num, 'link'))
+            if url in source_text:
+                continue
+
+            s = get_text(url)
+            source.append(s)
+            source_text[s['url']] = s['text']
+
+    for s in source_doc_links.values():
+        s.sort()
+
+    spans = list(enumerate(spans))
+
+    doc_spans = []
+    for num, (url, start, length) in spans:
+        highlight = url not in hide_transclusions
+        span_text = source_text[url]  # [start:start + length]
+        censor = url in censor_urls
+        new_text = ''
+        pos = start
+        for link_start, link_len, link_num, span_type in source_doc_links[url]:
+            link_end = link_start + link_len
+            if link_start >= start + length:
+                break
+            if link_end < start:
+                continue
+            open_tag = '<span class="xanadoclink link" id="xanalink{}">'.format(link_num)
+            link_span = (open_tag +
+                         escape(span_text[link_start:link_end]) +
+                         '</span>')
+            new_text += escape(span_text[pos:link_start]) + link_span
+            pos = link_end
+        new_text += escape(span_text[pos:start + length])
+        cur = xanadoc_span_html(num, new_text, url, start, length, highlight=highlight, censor=censor)
+        doc_spans.append(cur)
+
+    doc = ''.join(doc_spans)
+
+    for s in source:
+        text = s.pop('text')
+        if s['length'] > max_sourcedoc_size:
+            # print('{} > {}'.format(s['length'], max_sourcedoc_size))
+            continue
+        if s['url'] in hide_transclusions:
+            continue
+        source_spans = [(start, length, num, 'transclusion') for num, (url, start, length) in spans if url == s['url']]
+        source_spans += source_doc_links[s['url']]
+        source_spans.sort()
+
+        new_text = ''
+        pos = 0
+
+        for start, length, num, span_type in source_spans:
+            end = start + length
+            new_text += (escape(text[pos:start]) +
+                         span_html(span_type, num) +
+                         escape(text[start:end]) +
+                         '</span>')
+            pos = end
+        new_text += escape(text[pos:])
+        new_text = new_text.replace('\n', '<br/>\n')
+
+        s['text'] = new_text
+
+    return {
+        'source': source,
+        'doc': doc.replace('\n', '<br/>\n'),
+        'span_count': len(spans),
+        'link_count': len(two_facet_links),
+    }

sourcing/static/bootstrap

@@ -0,0 +1 @@
+/usr/share/javascript/bootstrap

sourcing/static/handlebars

@@ -0,0 +1 @@
+/usr/share/javascript/handlebars

sourcing/static/jquery

@@ -0,0 +1 @@
+/usr/share/javascript/jquery

sourcing/static/jquery-ui

@@ -0,0 +1 @@
+/usr/share/javascript/jquery-ui

sourcing/static/js/doc.js

@@ -0,0 +1,15 @@
+$(function() {
+    $("div#right").hide();
+    /* $("button#add-new-span").click(function() {
+        $("div#right").show();
+    }); */
+    $("a.source-doc-link").click(function(e) {
+        var link = $(e.target);
+        var title = link.text();
+        console.log("source doc link " + title);
+        $("div#source-doc-heading").text("Add span from " + title);
+        $("#newSpanModal").modal('hide');
+        $("div#right").show();
+        $("div#source-doc-body").load(link.data("url"));
+    });
+});

sourcing/static/js/sourcedoc.js

@@ -0,0 +1,49 @@
+function add_message() {
+  $("div#messages").text("whatever");
+}
+
+function show_selection(event) {
+  var range = window.getSelection().getRangeAt(0);
+  var start_element = range.startContainer.parentElement;
+  var end_element = range.endContainer.parentElement;
+
+  if ($(start_element).closest("div#right").prop("tagName") == 'DIV') {
+    return;
+  }
+
+  if (!start_element.hasAttribute('data-start') ||
+      !end_element.hasAttribute('data-start')) {
+    $("div#right").hide();
+    return;
+  }
+
+  var start = parseInt(start_element.getAttribute('data-start')) + range.startOffset;
+  var end = parseInt(end_element.getAttribute('data-start')) + range.endOffset;
+  var length = end - start;
+  if(length === 0) {
+    $("div#right").hide();
+    return;
+  }
+
+  $("#span").text(doc_url + ",start=" + start + ",length=" + length);
+
+  /*
+  $("div#right").show();
+  $("span#length").text(length);
+  $("input[name='offset']").val(start);
+  $("input[name='length']").val(length);
+  */
+}
+
+
+$(function() {
+  $("button#go").click(add_message);
+  $("div#right").hide();
+
+  $("body").mouseup(show_selection);
+
+  $("#show-span-selector").click(function(e) {
+    $("#span-selector").removeClass("hidden");
+    $(this).hide();
+  });
+});

sourcing/templates/edit.html

@@ -0,0 +1,17 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="edit " + doc.title %}
+
+{% set action=doc.edit_url %}
+{% set label="save" %}
+
+{% set fields %}
+{# 
+{{ render_field(form.filename) }}
+{{ render_field(form.db_price_per_character) }}
+{{ render_field(form.db_document_price) }}
+#}
+{{ render_field(form.text, rows=20) }}
+{% endset %}
+
+{% include "form/simple.html" %}

sourcing/templates/flash_msg.html

@@ -0,0 +1,10 @@
+{% with messages = get_flashed_messages() %}
+  {% if messages %}
+    {% for message in messages %}
+<div class="alert alert-success alert-dismissible" role="alert">
+  <button type="button" class="close" data-dismiss="alert"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+        {{ message }}
+</div>
+    {% endfor %}
+  {% endif %}
+{% endwith %}

sourcing/templates/foot.html

@@ -0,0 +1,11 @@
+<hr/>
+<footer>&copy; 2017 Project Xanadu</footer>
+</div>
+
+<script src="{{ url_for('static', filename='jquery/jquery.js') }}"></script>
+<script src="{{ url_for('static', filename='bootstrap/js/bootstrap.js') }}"></script>
+
+{{ scripts | safe }}
+
+</body>
+</html>

sourcing/templates/form/controls.html

@@ -0,0 +1,33 @@
+{% macro render_field(field) %}
+<div class="form-group">
+    {{ field.label() }}
+    {{ field(class="form-control", **kwargs) }}
+    {% if field.description %}
+        <span class="help">{{ field.description|safe }}</span>
+    {% endif %}
+</div>
+{% endmacro %}
+
+{% macro checkbox(field) %}
+<div class="form-group">
+    <label class="checkbox-inline">
+          {{ field() }}
+          {{ field.label.text }}
+    </label>
+    {% if field.description %}
+        <span class="help">{{ field.description|safe }}</span>
+    {% endif %}
+</div>
+{% endmacro %}
+
+{% macro radio(field) %}
+<div class="form-group">
+    <label class="radio-inline">
+          {{ field() }}
+          {{ field.label.text }}
+    </label>
+    {% if field.description %}
+        <span class="help">{{ field.description|safe }}</span>
+    {% endif %}
+</div>
+{% endmacro %}

sourcing/templates/form/errors.html

@@ -0,0 +1,9 @@
+{% if form.errors %}
+<ul class="errors">
+    {% for field in form %}
+        {% for error in form.errors[field.name] %}
+            <li>{{ field.label }}: {{ error }}</li>
+        {% endfor %}
+    {% endfor %}
+</ul>
+{% endif %}

sourcing/templates/form/main.html

@@ -0,0 +1,11 @@
+<div class="well">
+    {% include "form/errors.html" %}
+    <form action="{{action}}" method="{{method|default('post')}}" role="form">
+    {{ form.hidden_tag() }}
+    {{ fields | safe }}
+    <div class="form-group">
+        <button type="submit" class="btn btn-default">{{ label }}</button>
+    </div>
+    </form>
+</div>
+

sourcing/templates/form/simple.html

@@ -0,0 +1,10 @@
+{% include "head.html" %}
+
+<div class="row">
+    <div class="col-md-12">
+        <h1>{{ title }}</h1>
+        {% include "form/main.html" %}
+    </div>
+</div>
+
+{% include "foot.html" %}

sourcing/templates/head.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{ title | default("Xanadu") }}</title>
+
+<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap/css/bootstrap.css') }}">
+</head>
+
+<body>
+{% include "navbar.html" %}
+{{ dialogs | safe }}
+<div class="container">
+{% include "flash_msg.html" %}

sourcing/templates/home.html

@@ -0,0 +1,23 @@
+{% include "head.html" %}
+<h1>perma.pub</h1>
+
+<ul>
+{% for doc in docs %}
+<li>
+<a href="{{ doc.url }}">{{ doc.title() }}</a>
+by {{ doc.user.username }}
+&mdash; created {{ doc.created | datetime }}
+</li>
+{% endfor %}
+</ul>
+
+<p>
+<a href="{{ url_for('.new_sourcedoc') }}" class="btn btn-default">new source document</a>
+<a href="{{ url_for('.new_xanalink') }}" class="btn btn-default">new xanalink</a>
+<a href="{{ url_for('.new_xanadoc') }}" class="btn btn-default">new xanadoc</a>
+{# 
+<a href="#" class="btn btn-default">upload a source document</a>
+#}
+</p>
+
+{% include "foot.html" %}

sourcing/templates/login.html

@@ -0,0 +1,13 @@
+{% from "form/controls.html" import render_field, checkbox, submit %}
+
+{% set title="Login" %}
+{% set action=url_for('.login') %}
+{% set label="Sign in" %}
+
+{% set fields %}
+{{ render_field(form.user_or_email) }}
+{{ render_field(form.password) }}
+{{ checkbox(form.remember) }}
+{% endset %}
+
+{% include "form/simple.html" %}

sourcing/templates/mail/signup.txt

@@ -0,0 +1 @@
+You signed up for xanadu

sourcing/templates/navbar.html

@@ -0,0 +1,34 @@
+<nav class="navbar navbar-default">
+    <div class="container-fluid">
+        <div class="navbar-header">
+        	<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+				<span class="sr-only">Toggle navigation</span>
+				<span class="icon-bar"></span>
+				<span class="icon-bar"></span>
+				<span class="icon-bar"></span>
+            </button>
+            <a class="navbar-brand" href="{{ url_for('.home') }}">{{ config.SITE_NAME }}</a>
+        </div>
+
+		<div id="navbar" class="navbar-collapse collapse">
+			<ul class="nav navbar-nav">
+                {% for page in ('home', 'about', 'contact') %}
+                    <li{% if current_page == page %} class="active"{% endif %}><a href="{{ url_for('.' + page) }}">{{ page | title }}</a></li>
+                {% endfor %}
+			</ul>
+		    <ul class="nav navbar-nav navbar-right">
+                {% if current_user.is_authenticated %}
+                {% if current_user.admin %}
+                    <li><a href="{{ url_for('.admin') }}">admin</a></li>
+                {% endif %}
+				<li><a href="{{ url_for('.account_settings') }}">{{ current_user.username }}</a></li>
+				<li><a href="{{ url_for('.logout') }}">logout</a></li>
+                {% else %}
+				<li><a href="{{ url_for('.signup') }}">signup</a></li>
+				<li><a href="{{ url_for('.login') }}">login</a></li>
+                {% endif %}
+			</ul>
+		</div>
+    </div>
+</nav>
+

sourcing/templates/new.html

@@ -0,0 +1,15 @@
+{% from "form/controls.html" import render_field %}
+
+{% set action=url_for(request.endpoint) %}
+{% set label="save" %}
+
+{% set fields %}
+{# 
+{{ render_field(form.filename) }}
+{{ render_field(form.db_price_per_character) }}
+{{ render_field(form.db_document_price) }}
+#}
+{{ render_field(form.text, rows=20) }}
+{% endset %}
+
+{% include "form/simple.html" %}

sourcing/templates/signup.html

@@ -0,0 +1,13 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="Sign up" %}
+{% set action=url_for('.signup') %}
+{% set label="Sign up" %}
+
+{% set fields %}
+{{ render_field(form.username) }}
+{{ render_field(form.password) }}
+{{ render_field(form.email) }}
+{% endset %}
+
+{% include "form/simple.html" %}

sourcing/templates/source_doc.html

@@ -0,0 +1,33 @@
+{% set title = doc.filename %}
+{% set price = doc.document_price %}
+{% include "head.html" %}
+
+<style>
+div#text { font-family: Courier; }
+</style>
+
+<div class="row">
+  <div class="col-xs-12">
+    <h1>{{ title }}
+      {% if doc.user == current_user %}
+      <a href="{{ url_for('.edit_source_document',
+                          filename=doc.filename) }}" class="btn btn-default">edit</a>
+      {% endif %}
+    </h1>
+    <p><a href="{{ url_for('.home') }}">back to index</a></p>
+    <ul>
+    <li>character count: {{ '{:,d}'.format(doc.text | length) }}</li>
+    {% if price %}
+      <li>document price: {{ '{:,.2f}'.format(price) }} nanobucks</li>
+      <li>price per character: {{ '{:,.2f}'.format(doc.price_per_character) }} nanobucks</li>
+    {% endif %}
+    </ul>
+    <div class="well" id="text">
+    {%- for start, line in iter_lines(doc.get_text()) if line -%}
+      <p data-start="{{ start }}">{{ line }}</p>
+    {%- endfor -%}
+    </div>
+  </div>
+</div>
+
+{% include "foot.html" %}

sourcing/templates/user/account.html

@@ -0,0 +1,12 @@
+{% from "form/controls.html" import render_field %}
+
+{% set title="account settings" %}
+
+{% set action=url_for(".account_settings") %}
+{% set label="save" %}
+
+{% set fields %}
+{{ render_field(form.full_name) }}
+{% endset %}
+
+{% include "form/simple.html" %}

sourcing/templates/view.html

@@ -0,0 +1,34 @@
+{% set title = doc.title() %}
+{% include "head.html" %}
+
+<style>
+div#text { font-family: Courier; }
+</style>
+
+<div class="row">
+  <div class="col-xs-12">
+    <h1>{{ title }}
+      {% if doc.user == current_user %}
+      <a href="{{ doc.edit_url }}" class="btn btn-default">edit</a>
+      {% endif %}
+    </h1>
+    <p><a href="{{ url_for('.home') }}">back to index</a></p>
+    <div class="well" id="text">
+    {%- for start, line in iter_lines(doc.text) if line -%}
+      <p data-start="{{ start }}">{{ line }}</p>
+    {%- endfor -%}
+    </div>
+    {% if doc.type == 'sourcedoc' %}
+      <a href="#" id="show-span-selector" class="btn btn-default">show span selector</a>
+      <p id="span-selector" class="hidden">span: <span id="span"></span></p>
+      {% set scripts %}
+      <script>
+        var doc_url = '{{ doc.external_url }}';
+      </script>
+      <script src="{{ url_for('static', filename='js/sourcedoc.js') }}"></script>
+      {% endset %}
+    {% endif %}
+  </div>
+</div>
+
+{% include "foot.html" %}

sourcing/text.py

@@ -0,0 +1,21 @@
+import re
+import random
+
+re_newline = re.compile('\r?\n')
+
+def find_newlines(text):
+    return (m.end(0) for m in re_newline.finditer(text))
+
+def iter_lines(text):
+    start = 0
+    for m in re_newline.finditer(text):
+        end = m.end(0)
+        yield (start, text[start:end])
+        start = m.end(0)
+    if start < len(text) - 1:
+        yield (start, text[start:])
+
+def censor_text(text):
+    def random_chr():
+        return chr(random.randint(9728, 9983))
+    return ''.join(random_chr() if c.isalnum() else c for c in text)

sourcing/utils.py

@@ -0,0 +1,13 @@
+import humanize
+from datetime import date, timedelta
+
+def display_datetime(dt):
+    if dt is None:
+        return 'n/a'
+    today = date.today()
+    if today - dt.date() < timedelta(days=1):
+        return humanize.naturaltime(dt)
+    else:
+        return dt.strftime('%a, %d %b %Y')
+
+

sourcing/view.py

@@ -0,0 +1,232 @@
+from flask import (Blueprint, render_template, request, redirect, flash,
+                   url_for, abort, jsonify)
+from flask_login import (login_user, current_user, logout_user,
+                         login_required, LoginManager)
+from .forms import (LoginForm, SignupForm, AccountSettingsForm,
+                    UploadSourceDocForm, SourceDocForm, ItemForm)
+from .model import User, SourceDoc, Item, XanaDoc, XanaLink
+from .database import session
+from .text import iter_lines
+from werkzeug.debug.tbtools import get_current_traceback
+from jinja2 import evalcontextfilter, Markup
+from functools import wraps
+
+import re
+
+login_manager = LoginManager()
+login_manager.login_view = 'login'
+re_paragraph = re.compile(r'(?:\r\n|\r|\n){2,}')
+bp = Blueprint('view', __name__)
+
+def init_app(app):
+    login_manager.init_app(app)
+    app.register_blueprint(bp)
+
+    @app.template_filter()
+    @evalcontextfilter
+    def newline_html(eval_ctx, value):
+        return u'\n\n'.join(Markup(u'<p>') + p.replace('\n', Markup('<br>')) + Markup(u'</p>')
+                            for p in re_paragraph.split(value))
+
+@login_manager.user_loader
+def load_user(user_id):
+    return User.query.get(user_id)
+
+
+# where do we redirect after signup is complete
+view_after_signup = '.home'
+
+@bp.context_processor
+def inject_user():
+    return dict(current_user=current_user)
+
+def show_errors(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except Exception:
+            traceback = get_current_traceback(skip=1, show_hidden_frames=False,
+                                              ignore_system_exceptions=True)
+            return traceback.render_full().encode('utf-8', 'replace')
+    return wrapper
+
+@bp.route('/')
+def home():
+    titles = XanaLink.get_all_titles()
+    docs = Item.query.order_by(Item.created)
+    return render_template('home.html', docs=docs)
+
+@bp.route('/source_doc_upload', methods=["POST"])
+@show_errors
+def source_doc_upload():
+    f = request.files['sourcedoc_file']
+    text = f.read()
+    doc = SourceDoc(text=text, user=current_user, filename=f.filename)
+    session.add(doc)
+    session.commit()
+    flash('new source document uploaded')
+    return redirect(doc.url)
+
+@bp.route('/about')
+def about():
+    return render_template('about.html')
+
+@bp.route('/contact')
+def contact():
+    return render_template('contact.html')
+
+def redirect_to_home():
+    return redirect(url_for('.home'))
+
+@bp.route('/login', methods=['GET', 'POST'])
+def login():
+    form = LoginForm(next=request.args.get('next'))
+    if form.validate_on_submit():
+        login_user(form.user, remember=form.remember.data)
+        flash('Logged in successfully.')
+        return redirect(request.form.get('next') or url_for('.home'))
+    return render_template('login.html', form=form)
+
+@bp.route('/logout')
+def logout():
+    logout_user()
+    flash('You have been logged out.')
+    return redirect_to_home()
+
+@bp.route('/signup', methods=['GET', 'POST'])
+def signup():
+    form = SignupForm()
+    if not form.validate_on_submit():
+        return render_template('signup.html', form=form)
+
+    user = User(**form.data)
+    session.add(user)
+    session.commit()
+    flash('New account created.')
+    login_user(user)
+    return redirect(url_for(view_after_signup))
+
+def redirect_to_doc(doc):
+    return redirect(url_for('.view_document', hashid=doc.hashid))
+
+def get_source_doc(username, hashid):
+    doc = Item.get_by_hashid(hashid)
+    if doc and doc.user.username != username:
+        doc = None
+    return doc if doc else abort(404)
+
+def get_item(username, hashid):
+    doc = Item.get_by_hashid(hashid)
+    if doc and doc.user.username != username:
+        doc = None
+    return doc if doc else abort(404)
+
+@bp.route('/<username>/<hashid>')
+def view_item(username, hashid):
+    return render_template('view.html',
+                           doc=get_item(username, hashid),
+                           iter_lines=iter_lines)
+
+@bp.route('/<username>/<hashid>/edit', methods=['GET', 'POST'])
+def edit_item(username, hashid):
+    obj = get_item(current_user.username, hashid)
+    form = SourceDocForm(obj=obj)
+    if form.validate_on_submit():
+        form.populate_obj(obj)
+        session.add(obj)
+        session.commit()
+        flash('Changes to {} saved.'.format(obj.type))
+        return redirect(obj.url)
+    return render_template('edit.html', form=form, doc=obj)
+
+@bp.route('/source_doc_text/<int:source_doc_id>')
+def source_doc_text(source_doc_id):
+    doc = SourceDoc.query.get(source_doc_id)
+
+    return render_template('source_doc_text.html', doc=doc, iter_lines=iter_lines)
+
+@bp.route('/settings/account', methods=['GET', 'POST'])
+@login_required
+def account_settings():
+    form = AccountSettingsForm(obj=current_user)
+    if form.validate_on_submit():
+        form.populate_obj(current_user)
+        session.add(current_user)
+        session.commit()
+        flash('Account details updated.')
+        return redirect(url_for(request.endpoint))
+    return render_template('user/account.html', form=form)
+
+@bp.route('/new/sourcedoc', methods=['GET', 'POST'])
+@login_required
+def new_sourcedoc():
+    form = SourceDocForm()
+    if form.validate_on_submit():
+        doc = SourceDoc(user=current_user)
+        form.populate_obj(doc)
+        session.add(doc)
+        session.commit()
+        flash('New document saved.')
+        return redirect(doc.url)
+    return render_template('new.html', form=form, title='source document')
+
+@bp.route('/new/xanalink', methods=['GET', 'POST'])
+@login_required
+def new_xanalink():
+    form = ItemForm()
+    if form.validate_on_submit():
+        obj = XanaLink(user=current_user)
+        form.populate_obj(obj)
+        session.add(obj)
+        session.commit()
+        flash('New xanalink saved.')
+        return redirect(obj.url)
+    return render_template('new.html', form=form, title='xanalink')
+
+@bp.route('/new/xanadoc', methods=['GET', 'POST'])
+@login_required
+def new_xanadoc():
+    form = ItemForm()
+    if form.validate_on_submit():
+        obj = XanaLink(user=current_user)
+        form.populate_obj(obj)
+        session.add(obj)
+        session.commit()
+        flash('New xanalink saved.')
+        return redirect(obj.url)
+    return render_template('new.html', form=form, title='xanadoc')
+
+@bp.route('/edit/<filename>', methods=['GET', 'POST'])
+@login_required
+def edit_source_document(filename):
+    doc = get_source_doc(current_user.username, filename)
+    form = SourceDocForm(obj=doc)
+    if form.validate_on_submit():
+        form.populate_obj(doc)
+        session.add(doc)
+        session.commit()
+        flash('Changes to document saved.')
+        return redirect(doc.url)
+    return render_template('edit.html', form=form, doc=doc)
+
+@bp.route('/api/1/get/<username>/<filename>')
+def api_get_document(username, filename):
+    doc = get_source_doc(username, filename)
+    if not doc:
+        return abort(404)
+    ret = {
+        'username': username,
+        'filename': filename,
+        'character_count': len(doc.text),
+        'document_price': str(doc.document_price),
+        'price_per_character': str(doc.price_per_character),
+    }
+    return jsonify(ret)
+
+@bp.route('/all_titles')
+def get_all_titles():
+    titles = XanaLink.get_all_titles()
+    for k, v in titles.items():
+        print(from_external(k), v)
+    return ''

titles.py

@@ -0,0 +1,10 @@
+#!/usr/bin/python3
+
+from sourcing import create_app
+from sourcing.model import XanaLink, from_external
+
+if __name__ == "__main__":
+    app = create_app('config.default')
+
+    with app.test_client() as c:
+        c.get('/all_titles')