Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions internal/commands/richtext_format.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ func addRichTextFormatFlag(cmd *cobra.Command) {
}

// richTextToHTML is the one place a command turns rich-text input into the
// HTML Basecamp stores. With --format html the content is returned exactly as
// given. Otherwise it is read as Markdown, whatever it contains.
// HTML Basecamp stores. With --format html the content is returned as given.
// Otherwise it is read as Markdown, whatever it contains. Either way, a
// mention or file attachment copied from what the API returned is first
// reduced to the bare tag Basecamp expects (richtext.CollapseAttachments):
// sent back expanded, it corrupts the post.
//
// Content that holds HTML tags was sent as written before Markdown became the
// only default, so when --format was left unset and the Markdown keeps raw HTML
Expand All @@ -61,8 +64,9 @@ func richTextToHTML(cmd *cobra.Command, content string) string {
panic(cmd.CommandPath() + " reads rich text but has no --format flag; call addRichTextFormatFlag")
}
if flag.Value.String() == string(richTextHTML) {
return content
return richtext.CollapseAttachments(content)
}
content = richtext.CollapseAttachmentsInMarkdown(content)
if !flag.Changed && richtext.HasRichTextHTML(content) {
fmt.Fprintln(cmd.ErrOrStderr(), "Warning: content contains HTML tags and was read as Markdown, the default. "+
"Supported tags are kept and the Markdown around them is converted. "+
Expand Down
25 changes: 25 additions & 0 deletions internal/commands/richtext_format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,3 +270,28 @@ func TestNotesSetFormatHTMLSendsContentExactly(t *testing.T) {
require.NoError(t, json.Unmarshal([]byte(transport.last(t).Body), &body))
assert.Equal(t, passthroughHTML, body.Note.Content)
}

// expandedMention is a mention as the API returns it: the avatar figure inside
// the tag and the rendered mention in its content attribute.
const expandedMention = `<p dir="auto">Started by <bc-attachment sgid="BAh7mention" content-type="application/vnd.basecamp.mention" content="&lt;bc-mention gid=&quot;gid://bc3/Person/1&quot;&gt;
&lt;span&gt;Jason&lt;/span&gt;&lt;/bc-mention&gt;"><figure>
<img data-avatar-for-person-id="1" alt="Jason Fried" class="avatar" src="https://example.com/avatar" width="20" height="20">
<figcaption>
Jason
</figcaption>
</figure></bc-attachment> this week.</p>`

// Writing back a mention as the API returned it stored its avatar as content
// of its own, split the paragraph, and added a copy on every edit (#905).
// Basecamp's write form is the bare tag, in either format.
func TestCommentsUpdateCollapsesAMentionCopiedFromTheAPI(t *testing.T) {
want := `<p dir="auto">Started by <bc-attachment sgid="BAh7mention" content-type="application/vnd.basecamp.mention"></bc-attachment> this week.</p>`
for _, format := range []string{"html", "markdown"} {
t.Run(format, func(t *testing.T) {
transport := &mockCommentWriteTransport{}
_, err := runCommentWithStderr(t, transport, expandedMention+"\n", "update", "1234", "-", "--format", format)
require.NoError(t, err)
assert.Equal(t, want, sentContent(t, transport))
})
}
}
180 changes: 180 additions & 0 deletions internal/richtext/attachments.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
package richtext

import (
"bytes"
"regexp"
"sort"
"strings"

"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/text"
"golang.org/x/net/html"
)

// reTagAttribute matches one attribute of a tag: its name, and its value if it
// has one, quoted or not.
var reTagAttribute = regexp.MustCompile(`([^\s"'=<>/]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>` + "`" + `]+))?`)

// CollapseAttachments returns html with every <bc-attachment> that names a
// record (it has an sgid) reduced to the form Basecamp documents for writing
// one: the tag with nothing inside it and without its content attribute.
//
// Basecamp returns a mention or a file expanded: the person's avatar or the
// file's preview as a <figure> inside the tag, and for a mention its rendered
// HTML in content. Written back as returned, the figure is stored as content
// of its own: it breaks the paragraph around a mention apart, and each edit
// adds another copy. Basecamp expands the bare tag again when it is read, so
// collapsing it loses nothing. An attachment without an sgid carries its
// content itself (a horizontal rule, say) and is left as written, and so is
// one that is self-closing or never closed.
func CollapseAttachments(s string) string {
if !hasAttachmentTag(s) {
return s
}
return collapseAttachments(s, attachmentTagsIn([]byte(s), []text.Segment{text.NewSegment(0, len(s))}))
}

// CollapseAttachmentsInMarkdown is CollapseAttachments for Markdown. Only an
// attachment tag the Markdown renderer writes as markup counts: one written
// in code, escaped with a backslash, or in an HTML block the renderer shows
// as text is left as written.
func CollapseAttachmentsInMarkdown(md string) string {
if !hasAttachmentTag(md) {
return md
}
return collapseAttachments(md, markdownAttachmentTags(md))
}

func hasAttachmentTag(s string) bool {
return strings.Contains(strings.ToLower(s), "<bc-attachment")
}

// attachmentTag is one <bc-attachment> tag: where it starts and ends in the
// string it was read from, which kind of tag it is, and the tag as read. A
// tag can span lines whose prefixes (a blockquote's "> ") are not part of it,
// so the tag as read can differ from the source between start and end.
type attachmentTag struct {
start, end int
kind html.TokenType
raw string
}

// collapseAttachments collapses the attachment elements tags pair up in s.
// An opening tag pairs with the next tag when that tag closes it. One that is
// never closed, or is followed by another attachment first, is left as
// written, and so is a self-closing one.
func collapseAttachments(s string, tags []attachmentTag) string {
var b strings.Builder
last := 0
for i := 0; i+1 < len(tags); i++ {
open, next := tags[i], tags[i+1]
if open.kind != html.StartTagToken || next.kind != html.EndTagToken {
continue
}
b.WriteString(s[last:open.start])
b.WriteString(collapseAttachment(s[open.start:next.end], open.raw))
last = next.end
i++
}
b.WriteString(s[last:])
return b.String()
}

// attachmentTagsIn reads the segments of source, joined, as HTML and
// returns their attachment tags, placed in source. A tokenizer reads them, so
// markup inside a comment or inside another tag's attribute value is not
// taken for a tag.
func attachmentTagsIn(source []byte, segments []text.Segment) []attachmentTag {
var joined []byte
var at []int // at[i] is where joined[i] is in source
for _, seg := range segments {
for range seg.Padding {
joined = append(joined, ' ')
at = append(at, seg.Start)
}
for i := seg.Start; i < seg.Stop; i++ {
joined = append(joined, source[i])
at = append(at, i)
}
}

var tags []attachmentTag
z := html.NewTokenizer(bytes.NewReader(joined))
for pos := 0; ; {
kind := z.Next()
if kind == html.ErrorToken {
return tags
}
size := len(z.Raw())
isTag := kind == html.StartTagToken || kind == html.EndTagToken || kind == html.SelfClosingTagToken
if name, _ := z.TagName(); isTag && bytes.Equal(name, []byte("bc-attachment")) {
tags = append(tags, attachmentTag{
start: at[pos], end: at[pos+size-1] + 1, kind: kind, raw: string(joined[pos : pos+size]),
})
}
pos += size
}
}

// markdownAttachmentTags returns the attachment tags in md that its renderer
// writes as markup: inline raw HTML, and the HTML blocks it passes through
// (isRichTextTag, as renderHTMLBlock decides), each read from its segments
// as the renderer reads it. Code, backslash escapes and the HTML blocks it
// escapes hold none.
func markdownAttachmentTags(md string) []attachmentTag {
source := []byte(md)
doc := mdConverter.Parser().Parse(text.NewReader(source))

var tags []attachmentTag
read := func(segments []text.Segment) {
tags = append(tags, attachmentTagsIn(source, segments)...)
}
_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
switch node := n.(type) {
case *ast.RawHTML:
read(node.Segments.Sliced(0, node.Segments.Len()))
case *ast.HTMLBlock:
lines := node.Lines()
if isRichTextTag(lines.Value(source)) {
segments := lines.Sliced(0, lines.Len())
if node.HasClosure() {
segments = append(segments, node.ClosureLine)
}
read(segments)
}
return ast.WalkSkipChildren, nil
}
return ast.WalkContinue, nil
})
sort.Slice(tags, func(i, j int) bool { return tags[i].start < tags[j].start })
return tags
}

// collapseAttachment reduces one <bc-attachment> element to its write form,
// or returns it unchanged when it names no record.
func collapseAttachment(element, open string) string {
attrs := reTagAttribute.FindAllString(open[len("<bc-attachment"):len(open)-1], -1)

kept := make([]string, 0, len(attrs))
hasSGID := false
for _, attr := range attrs {
name := strings.ToLower(attr)
if i := strings.IndexAny(name, " \t\n\r="); i >= 0 {
name = name[:i]
}
switch name {
case "sgid":
hasSGID = true
case "content":
continue
}
kept = append(kept, attr)
}
if !hasSGID {
return element
}
return "<bc-attachment " + strings.Join(kept, " ") + "></bc-attachment>"
}
160 changes: 160 additions & 0 deletions internal/richtext/attachments_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package richtext

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestCollapseAttachmentsReducesAnExpandedMention(t *testing.T) {
input := `<p>Hi <bc-attachment sgid="BAh7" content-type="application/vnd.basecamp.mention" content="&lt;bc-mention&gt;
&lt;span&gt;Jo&lt;/span&gt;&lt;/bc-mention&gt;"><figure>
<img class="avatar" alt="Jo" src="https://example.com/a">
<figcaption>Jo</figcaption>
</figure></bc-attachment> there.</p>`

assert.Equal(t,
`<p>Hi <bc-attachment sgid="BAh7" content-type="application/vnd.basecamp.mention"></bc-attachment> there.</p>`,
CollapseAttachments(input))
}

func TestCollapseAttachmentsKeepsAFileAttachmentsOwnAttributes(t *testing.T) {
input := `<bc-attachment sgid="BAh8" content-type="image/png" filename="shot.png" caption="The bug" presentation="gallery"><figure><img src="https://example.com/shot.png"><figcaption>The bug</figcaption></figure></bc-attachment>`

assert.Equal(t,
`<bc-attachment sgid="BAh8" content-type="image/png" filename="shot.png" caption="The bug" presentation="gallery"></bc-attachment>`,
CollapseAttachments(input))
}

func TestCollapseAttachmentsLeavesWhatNamesNoRecord(t *testing.T) {
for _, input := range []string{
`<bc-attachment content-type="vnd.basecamp.horizontal-rule.html" content="&lt;hr&gt;"><figure><hr></figure></bc-attachment>`,
`<bc-attachment sgid="BAh9" content-type="application/pdf"/>`,
`<bc-attachment sgid="BAh9" content-type="application/vnd.basecamp.mention"></bc-attachment>`,
`<p>no attachments</p>`,
} {
assert.Equal(t, input, CollapseAttachments(input))
}
}

func TestCollapseAttachmentsInMarkdownLeavesCode(t *testing.T) {
expanded := `<bc-attachment sgid="BAh7" content-type="application/vnd.basecamp.mention"><figure><img alt="Jo"></figure></bc-attachment>`
bare := `<bc-attachment sgid="BAh7" content-type="application/vnd.basecamp.mention"></bc-attachment>`
input := "Hi " + expanded + " there.\n\n" +
"`" + expanded + "`\n\n" +
"```html\n" + expanded + "\n```\n\n" +
" " + expanded + "\n"

assert.Equal(t,
"Hi "+bare+" there.\n\n"+
"`"+expanded+"`\n\n"+
"```html\n"+expanded+"\n```\n\n"+
" "+expanded+"\n",
CollapseAttachmentsInMarkdown(input))
}

// A self-closing attachment is not the opening tag of the next one: reading it
// that way dropped everything up to the next closing tag.
func TestCollapseAttachmentsSkipsASelfClosingAttachment(t *testing.T) {
input := `<bc-attachment sgid="file"/>KEEP THIS TEXT<bc-attachment sgid="person"><figure>Jo</figure></bc-attachment>`

assert.Equal(t,
`<bc-attachment sgid="file"/>KEEP THIS TEXT<bc-attachment sgid="person"></bc-attachment>`,
CollapseAttachments(input))
assert.Equal(t,
`<bc-attachment sgid="file"/>KEEP THIS TEXT<bc-attachment sgid="person"></bc-attachment>`,
CollapseAttachmentsInMarkdown(input))
}

func TestCollapseAttachmentsReadsAQuotedGreaterThan(t *testing.T) {
input := `<bc-attachment sgid="x" caption="a > b" title='c > d'><figure>a > b</figure></bc-attachment>`

assert.Equal(t,
`<bc-attachment sgid="x" caption="a > b" title='c > d'></bc-attachment>`,
CollapseAttachments(input))
}

func TestCollapseAttachmentsLeavesAnUnclosedAttachment(t *testing.T) {
for _, input := range []string{
`<p>Hi <bc-attachment sgid="x"><figure>Jo</figure></p>`,
`<p>Hi <bc-attachment sgid="x" caption="never closed`,
`<bc-attachment-like sgid="x"><figure>Jo</figure></bc-attachment-like>`,
} {
assert.Equal(t, input, CollapseAttachments(input))
}

assert.Equal(t,
`<bc-attachment sgid="a">KEEP <bc-attachment sgid="b"></bc-attachment>`,
CollapseAttachments(`<bc-attachment sgid="a">KEEP <bc-attachment sgid="b"><figure>Jo</figure></bc-attachment>`))
}

// Lowering a string for a case-blind search must not move its bytes: 'İ'
// lowers to three bytes from two.
func TestCollapseAttachmentsAfterTextThatLowersLonger(t *testing.T) {
input := `<p>İİ <BC-ATTACHMENT sgid="x"><figure>Jo</figure></BC-ATTACHMENT> ok</p>`

assert.Equal(t, `<p>İİ <bc-attachment sgid="x"></bc-attachment> ok</p>`, CollapseAttachments(input))
}

// A backslash makes a tag literal Markdown text, so neither an escaped
// opening tag nor an escaped closing tag is markup.
func TestCollapseAttachmentsInMarkdownLeavesEscapedTags(t *testing.T) {
for _, input := range []string{
`Example: \<bc-attachment sgid="x">KEEP THIS TEXT\</bc-attachment>`,
`Example: <bc-attachment sgid="x">KEEP THIS TEXT\</bc-attachment>`,
} {
assert.Equal(t, input, CollapseAttachmentsInMarkdown(input))
}

assert.Equal(t,
`Example: \\<bc-attachment sgid="x"></bc-attachment>`,
CollapseAttachmentsInMarkdown(`Example: \\<bc-attachment sgid="x"><figure>Jo</figure></bc-attachment>`))
}

// Markup inside an HTML comment is not markup: reading it as an attachment
// deleted the visible content between two comments.
func TestCollapseAttachmentsLeavesTagsInComments(t *testing.T) {
input := `<!-- <bc-attachment sgid="x"> --><p>KEEP THIS TEXT</p><!-- </bc-attachment> -->`

assert.Equal(t, input, CollapseAttachments(input))
assert.Equal(t, input, CollapseAttachmentsInMarkdown(input))

assert.Equal(t,
`<!-- note --><bc-attachment sgid="x"></bc-attachment><!-- unterminated <bc-attachment sgid="y"><figure>Jo</figure></bc-attachment>`,
CollapseAttachments(`<!-- note --><bc-attachment sgid="x"><figure>Jo</figure></bc-attachment><!-- unterminated <bc-attachment sgid="y"><figure>Jo</figure></bc-attachment>`))
}

// Attachment markup inside another tag's attribute value is the value's text,
// whether it would open an element or close a real one too early.
func TestCollapseAttachmentsIgnoresTagsInAttributeValues(t *testing.T) {
quoted := `<span title='<bc-attachment sgid="x">'>KEEP THIS TEXT</span><span title='</bc-attachment>'>after</span>`
assert.Equal(t, quoted, CollapseAttachments(quoted))
assert.Equal(t, quoted, CollapseAttachmentsInMarkdown(quoted))

alt := `<bc-attachment sgid="x"><figure><img alt="</bc-attachment>" src="a.png"><figcaption>Jo</figcaption></figure></bc-attachment>`
assert.Equal(t, `<bc-attachment sgid="x"></bc-attachment>`, CollapseAttachments(alt))
assert.Equal(t, `<bc-attachment sgid="x"></bc-attachment>`, CollapseAttachmentsInMarkdown(alt))
}

// The Markdown renderer shows an HTML block it does not support as escaped
// text, so an attachment written in one is text too.
func TestCollapseAttachmentsInMarkdownLeavesEscapedHTMLBlocks(t *testing.T) {
input := `<section><bc-attachment sgid="x">KEEP THIS TEXT</bc-attachment></section>`

assert.Equal(t, input, CollapseAttachmentsInMarkdown(input))
assert.Contains(t, MarkdownToHTML(CollapseAttachmentsInMarkdown(input)), "KEEP THIS TEXT")
}

// A tag split across the lines of a blockquote is read without the "> " that
// starts each line, as the renderer reads it: with the prefix put back, its
// '>' ended the tag early and the attributes after it were dropped.
func TestCollapseAttachmentsInMarkdownReadsATagAcrossBlockquoteLines(t *testing.T) {
input := "> Hi <bc-attachment sgid=\"x\"\n> content-type=\"image/png\" caption=\"keep\"><figure>Jo</figure></bc-attachment> bye\n"

collapsed := CollapseAttachmentsInMarkdown(input)
assert.Equal(t,
"> Hi <bc-attachment sgid=\"x\" content-type=\"image/png\" caption=\"keep\"></bc-attachment> bye\n",
collapsed)
assert.Contains(t, MarkdownToHTML(collapsed),
`<bc-attachment sgid="x" content-type="image/png" caption="keep"></bc-attachment>`)
}
Loading