Add data layer

This includes config changes, collections, posts, some post rendering
funcs, and actual database connection when the server starts up.
This commit is contained in:
Matt Baer 2018-10-16 22:31:27 -04:00
parent f7430fb8bc
commit 0c1e1dd57e
8 changed files with 2616 additions and 8 deletions

28
app.go
View file

@ -1,6 +1,7 @@
package writefreely
import (
"database/sql"
"flag"
"fmt"
_ "github.com/go-sql-driver/mysql"
@ -21,6 +22,7 @@ const (
type app struct {
router *mux.Router
db *datastore
cfg *config.Config
keys *keychain
sessionStore *sessions.CookieStore
@ -62,11 +64,33 @@ func Serve() {
// Initialize modules
app.sessionStore = initSession(app)
// Check database configuration
if app.cfg.Database.User == "" || app.cfg.Database.Password == "" {
log.Error("Database user or password not set.")
os.Exit(1)
}
if app.cfg.Database.Host == "" {
app.cfg.Database.Host = "localhost"
}
if app.cfg.Database.Database == "" {
app.cfg.Database.Database = "writeas"
}
log.Info("Connecting to database...")
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=true", app.cfg.Database.User, app.cfg.Database.Password, app.cfg.Database.Host, app.cfg.Database.Port, app.cfg.Database.Database))
if err != nil {
log.Error("\n%s\n", err)
os.Exit(1)
}
app.db = &datastore{db}
defer shutdown(app)
app.db.SetMaxOpenConns(50)
r := mux.NewRouter()
handler := NewHandler(app.sessionStore)
// Handle app routes
initRoutes(handler, r, app.cfg)
initRoutes(handler, r, app.cfg, app.db)
// Handle static files
fs := http.FileServer(http.Dir(staticDir))
@ -92,4 +116,6 @@ func Serve() {
}
func shutdown(app *app) {
log.Info("Closing database connection...")
app.db.Close()
}

69
collections.go Normal file
View file

@ -0,0 +1,69 @@
package writefreely
import (
"database/sql"
)
type (
Collection struct {
ID int64 `datastore:"id" json:"-"`
Alias string `datastore:"alias" schema:"alias" json:"alias"`
Title string `datastore:"title" schema:"title" json:"title"`
Description string `datastore:"description" schema:"description" json:"description"`
Direction string `schema:"dir" json:"dir,omitempty"`
Language string `schema:"lang" json:"lang,omitempty"`
StyleSheet string `datastore:"style_sheet" schema:"style_sheet" json:"style_sheet"`
Script string `datastore:"script" schema:"script" json:"script,omitempty"`
Public bool `datastore:"public" json:"public"`
Visibility collVisibility `datastore:"private" json:"-"`
Format string `datastore:"format" json:"format,omitempty"`
Views int64 `json:"views"`
OwnerID int64 `datastore:"owner_id" json:"-"`
PublicOwner bool `datastore:"public_owner" json:"-"`
PreferSubdomain bool `datastore:"prefer_subdomain" json:"-"`
Domain string `datastore:"domain" json:"domain,omitempty"`
IsDomainActive bool `datastore:"is_active" json:"-"`
IsSecure bool `datastore:"is_secure" json:"-"`
CustomHandle string `datastore:"handle" json:"-"`
Email string `json:"email,omitempty"`
URL string `json:"url,omitempty"`
app *app
}
CollectionObj struct {
Collection
TotalPosts int `json:"total_posts"`
Owner *User `json:"owner,omitempty"`
Posts *[]PublicPost `json:"posts,omitempty"`
}
SubmittedCollection struct {
// Data used for updating a given collection
ID int64
OwnerID uint64
// Form helpers
PreferURL string `schema:"prefer_url" json:"prefer_url"`
Privacy int `schema:"privacy" json:"privacy"`
Pass string `schema:"password" json:"password"`
Federate bool `schema:"federate" json:"federate"`
MathJax bool `schema:"mathjax" json:"mathjax"`
Handle string `schema:"handle" json:"handle"`
// Actual collection values updated in the DB
Alias *string `schema:"alias" json:"alias"`
Title *string `schema:"title" json:"title"`
Description *string `schema:"description" json:"description"`
StyleSheet *sql.NullString `schema:"style_sheet" json:"style_sheet"`
Script *sql.NullString `schema:"script" json:"script"`
Visibility *int `schema:"visibility" json:"public"`
Format *sql.NullString `schema:"format" json:"format"`
PreferSubdomain *bool `schema:"prefer_subdomain" json:"prefer_subdomain"`
Domain *sql.NullString `schema:"domain" json:"domain"`
}
CollectionFormat struct {
Format string
}
)
// collVisibility represents the visibility level for the collection.
type collVisibility int

View file

@ -15,11 +15,12 @@ type (
}
DatabaseCfg struct {
Type string `ini:"type"`
User string `ini:"username"`
Pass string `ini:"password"`
Host string `ini:"host"`
Port int `ini:"port"`
Type string `ini:"type"`
User string `ini:"username"`
Password string `ini:"password"`
Database string `ini:"database"`
Host string `ini:"host"`
Port int `ini:"port"`
}
AppCfg struct {

2180
database.go Normal file

File diff suppressed because it is too large Load diff

View file

@ -7,5 +7,24 @@ import (
// Commonly returned HTTP errors
var (
ErrInternalCookieSession = impart.HTTPError{http.StatusInternalServerError, "Could not get cookie session."}
ErrBadAccessToken = impart.HTTPError{http.StatusUnauthorized, "Invalid access token."}
ErrNoAccessToken = impart.HTTPError{http.StatusBadRequest, "Authorization token required."}
ErrForbiddenCollection = impart.HTTPError{http.StatusForbidden, "You don't have permission to add to this collection."}
ErrUnauthorizedEditPost = impart.HTTPError{http.StatusUnauthorized, "Invalid editing credentials."}
ErrUnauthorizedGeneral = impart.HTTPError{http.StatusUnauthorized, "You don't have permission to do that."}
ErrInternalGeneral = impart.HTTPError{http.StatusInternalServerError, "The humans messed something up. They've been notified."}
ErrCollectionPageNotFound = impart.HTTPError{http.StatusNotFound, "Collection page doesn't exist."}
ErrPostNotFound = impart.HTTPError{Status: http.StatusNotFound, Message: "Post not found."}
ErrPostUnpublished = impart.HTTPError{Status: http.StatusGone, Message: "Post unpublished by author."}
ErrPostFetchError = impart.HTTPError{Status: http.StatusInternalServerError, Message: "We encountered an error getting the post. The humans have been alerted."}
ErrUserNotFound = impart.HTTPError{http.StatusNotFound, "User doesn't exist."}
)
// Post operation errors
var (
ErrPostNoUpdatableVals = impart.HTTPError{http.StatusBadRequest, "Supply some properties to update."}
)

135
postrender.go Normal file
View file

@ -0,0 +1,135 @@
package writefreely
import (
"bytes"
"github.com/microcosm-cc/bluemonday"
stripmd "github.com/writeas/go-strip-markdown"
"github.com/writeas/saturday"
"html"
"html/template"
"regexp"
"strings"
"unicode"
"unicode/utf8"
)
var (
blockReg = regexp.MustCompile("<(ul|ol|blockquote)>\n")
endBlockReg = regexp.MustCompile("</([a-z]+)>\n</(ul|ol|blockquote)>")
youtubeReg = regexp.MustCompile("(https?://www.youtube.com/embed/[a-zA-Z0-9\\-_]+)(\\?[^\t\n\f\r \"']+)?")
titleElementReg = regexp.MustCompile("</?h[1-6]>")
hashtagReg = regexp.MustCompile(`#([\p{L}\p{M}\d]+)`)
markeddownReg = regexp.MustCompile("<p>(.+)</p>")
)
func (p *Post) formatContent(c *Collection, isOwner bool) {
baseURL := c.CanonicalURL()
if isOwner {
baseURL = "/" + c.Alias + "/"
}
newCon := hashtagReg.ReplaceAllFunc([]byte(p.Content), func(b []byte) []byte {
// Ensure we only replace "hashtags" that have already been extracted.
// `hashtagReg` catches everything, including any hash on the end of a
// URL, so we rely on p.Tags as the final word on whether or not to link
// a tag.
for _, t := range p.Tags {
if string(b) == "#"+t {
return bytes.Replace(b, []byte("#"+t), []byte("<a href=\""+baseURL+"tag:"+t+"\" class=\"hashtag\"><span>#</span><span class=\"p-category\">"+t+"</span></a>"), -1)
}
}
return b
})
p.HTMLTitle = template.HTML(applyBasicMarkdown([]byte(p.Title.String)))
p.HTMLContent = template.HTML(applyMarkdown([]byte(newCon)))
if exc := strings.Index(string(newCon), "<!--more-->"); exc > -1 {
p.HTMLExcerpt = template.HTML(applyMarkdown([]byte(newCon[:exc])))
}
}
func (p *PublicPost) formatContent(isOwner bool) {
p.Post.formatContent(&p.Collection.Collection, isOwner)
}
func applyMarkdown(data []byte) string {
return applyMarkdownSpecial(data, false)
}
func applyMarkdownSpecial(data []byte, skipNoFollow bool) string {
mdExtensions := 0 |
blackfriday.EXTENSION_TABLES |
blackfriday.EXTENSION_FENCED_CODE |
blackfriday.EXTENSION_AUTOLINK |
blackfriday.EXTENSION_STRIKETHROUGH |
blackfriday.EXTENSION_SPACE_HEADERS |
blackfriday.EXTENSION_AUTO_HEADER_IDS
htmlFlags := 0 |
blackfriday.HTML_USE_SMARTYPANTS |
blackfriday.HTML_SMARTYPANTS_DASHES
// Generate Markdown
md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
// Strip out bad HTML
policy := getSanitizationPolicy()
policy.RequireNoFollowOnLinks(!skipNoFollow)
outHTML := string(policy.SanitizeBytes(md))
// Strip newlines on certain block elements that render with them
outHTML = blockReg.ReplaceAllString(outHTML, "<$1>")
outHTML = endBlockReg.ReplaceAllString(outHTML, "</$1></$2>")
// Remove all query parameters on YouTube embed links
// TODO: make this more specific. Taking the nuclear approach here to strip ?autoplay=1
outHTML = youtubeReg.ReplaceAllString(outHTML, "$1")
return outHTML
}
func applyBasicMarkdown(data []byte) string {
mdExtensions := 0 |
blackfriday.EXTENSION_STRIKETHROUGH |
blackfriday.EXTENSION_SPACE_HEADERS |
blackfriday.EXTENSION_HEADER_IDS
htmlFlags := 0 |
blackfriday.HTML_SKIP_HTML |
blackfriday.HTML_USE_SMARTYPANTS |
blackfriday.HTML_SMARTYPANTS_DASHES
// Generate Markdown
md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions)
// Strip out bad HTML
policy := bluemonday.UGCPolicy()
policy.AllowAttrs("class", "id").Globally()
outHTML := string(policy.SanitizeBytes(md))
outHTML = markeddownReg.ReplaceAllString(outHTML, "$1")
outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace)
return outHTML
}
func postTitle(content, friendlyId string) string {
const maxTitleLen = 80
// Strip HTML tags with bluemonday's StrictPolicy, then unescape the HTML
// entities added in by sanitizing the content.
content = html.UnescapeString(bluemonday.StrictPolicy().Sanitize(content))
content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace)
eol := strings.IndexRune(content, '\n')
blankLine := strings.Index(content, "\n\n")
if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen {
return strings.TrimSpace(content[:blankLine])
} else if utf8.RuneCountInString(content) <= maxTitleLen {
return content
}
return friendlyId
}
func getSanitizationPolicy() *bluemonday.Policy {
policy := bluemonday.UGCPolicy()
policy.AllowAttrs("src", "style").OnElements("iframe", "video")
policy.AllowAttrs("frameborder", "width", "height").Matching(bluemonday.Integer).OnElements("iframe")
policy.AllowAttrs("allowfullscreen").OnElements("iframe")
policy.AllowAttrs("controls", "loop", "muted", "autoplay").OnElements("video")
policy.AllowAttrs("target").OnElements("a")
policy.AllowAttrs("style", "class", "id").Globally()
policy.AllowURLSchemes("http", "https", "mailto", "xmpp")
return policy
}

178
posts.go Normal file
View file

@ -0,0 +1,178 @@
package writefreely
import (
"github.com/guregu/null"
"github.com/guregu/null/zero"
"github.com/kylemcc/twitter-text-go/extract"
"github.com/writeas/monday"
"github.com/writeas/slug"
"github.com/writeas/web-core/converter"
"github.com/writeas/web-core/parse"
"github.com/writeas/web-core/tags"
"html/template"
"regexp"
"time"
)
const (
// Post ID length bounds
minIDLen = 10
maxIDLen = 10
userPostIDLen = 10
postIDLen = 10
postMetaDateFormat = "2006-01-02 15:04:05"
)
type (
AuthenticatedPost struct {
ID string `json:"id" schema:"id"`
*SubmittedPost
}
// SubmittedPost represents a post supplied by a client for publishing or
// updating. Since Title and Content can be updated to "", they are
// pointers that can be easily tested to detect changes.
SubmittedPost struct {
Slug *string `json:"slug" schema:"slug"`
Title *string `json:"title" schema:"title"`
Content *string `json:"body" schema:"body"`
Font string `json:"font" schema:"font"`
IsRTL converter.NullJSONBool `json:"rtl" schema:"rtl"`
Language converter.NullJSONString `json:"lang" schema:"lang"`
Created *string `json:"created" schema:"created"`
// [{ "medium": "ev" }, { "twitter": "ilikebeans" }]
Crosspost []map[string]string `json:"crosspost" schema:"crosspost"`
}
// Post represents a post as found in the database.
Post struct {
ID string `db:"id" json:"id"`
Slug null.String `db:"slug" json:"slug,omitempty"`
Font string `db:"text_appearance" json:"appearance"`
Language zero.String `db:"language" json:"language"`
RTL zero.Bool `db:"rtl" json:"rtl"`
Privacy int64 `db:"privacy" json:"-"`
OwnerID null.Int `db:"owner_id" json:"-"`
CollectionID null.Int `db:"collection_id" json:"-"`
PinnedPosition null.Int `db:"pinned_position" json:"-"`
Created time.Time `db:"created" json:"created"`
Updated time.Time `db:"updated" json:"updated"`
ViewCount int64 `db:"view_count" json:"-"`
EmbedViewCount int64 `db:"embed_view_count" json:"-"`
Title zero.String `db:"title" json:"title"`
HTMLTitle template.HTML `db:"title" json:"-"`
Content string `db:"content" json:"body"`
HTMLContent template.HTML `db:"content" json:"-"`
HTMLExcerpt template.HTML `db:"content" json:"-"`
Tags []string `json:"tags"`
Images []string `json:"images,omitempty"`
OwnerName string `json:"owner,omitempty"`
}
// PublicPost holds properties for a publicly returned post, i.e. a post in
// a context where the viewer may not be the owner. As such, sensitive
// metadata for the post is hidden and properties supporting the display of
// the post are added.
PublicPost struct {
*Post
IsSubdomain bool `json:"-"`
IsTopLevel bool `json:"-"`
Domain string `json:"-"`
DisplayDate string `json:"-"`
Views int64 `json:"views"`
Owner *PublicUser `json:"-"`
IsOwner bool `json:"-"`
Collection *CollectionObj `json:"collection,omitempty"`
}
AnonymousAuthPost struct {
ID string `json:"id"`
Token string `json:"token"`
}
ClaimPostRequest struct {
*AnonymousAuthPost
CollectionAlias string `json:"collection"`
CreateCollection bool `json:"create_collection"`
// Generated properties
Slug string `json:"-"`
}
ClaimPostResult struct {
ID string `json:"id,omitempty"`
Code int `json:"code,omitempty"`
ErrorMessage string `json:"error_msg,omitempty"`
Post *PublicPost `json:"post,omitempty"`
}
)
func (p *Post) processPost() PublicPost {
res := &PublicPost{Post: p, Views: 0}
res.Views = p.ViewCount
// TODO: move to own function
loc := monday.FuzzyLocale(p.Language.String)
res.DisplayDate = monday.Format(p.Created, monday.LongFormatsByLocale[loc], loc)
return *res
}
// TODO: merge this into getSlugFromPost or phase it out
func getSlug(title, lang string) string {
return getSlugFromPost("", title, lang)
}
func getSlugFromPost(title, body, lang string) string {
if title == "" {
title = postTitle(body, body)
}
title = parse.PostLede(title, false)
// Truncate lede if needed
title, _ = parse.TruncToWord(title, 80)
if lang != "" && len(lang) == 2 {
return slug.MakeLang(title, lang)
}
return slug.Make(title)
}
// isFontValid returns whether or not the submitted post's appearance is valid.
func (p *SubmittedPost) isFontValid() bool {
validFonts := map[string]bool{
"norm": true,
"sans": true,
"mono": true,
"wrap": true,
"code": true,
}
if _, valid := validFonts[p.Font]; valid {
return true
}
return false
}
func (p *Post) extractData() {
p.Tags = tags.Extract(p.Content)
p.extractImages()
}
var imageURLRegex = regexp.MustCompile(`(?i)^https?:\/\/[^ ]*\.(gif|png|jpg|jpeg)$`)
func (p *Post) extractImages() {
matches := extract.ExtractUrls(p.Content)
urls := map[string]bool{}
for i := range matches {
u := matches[i].Text
if !imageURLRegex.MatchString(u) {
continue
}
urls[u] = true
}
resURLs := make([]string, 0)
for k := range urls {
resURLs = append(resURLs, k)
}
p.Images = resURLs
}

View file

@ -7,7 +7,7 @@ import (
"strings"
)
func initRoutes(handler *Handler, r *mux.Router, cfg *config.Config) {
func initRoutes(handler *Handler, r *mux.Router, cfg *config.Config, db *datastore) {
isSingleUser := !cfg.App.MultiUser
// Write.as router