diff --git a/internal/commands/richtext_format.go b/internal/commands/richtext_format.go index ea84cdbf4..dd1e89d87 100644 --- a/internal/commands/richtext_format.go +++ b/internal/commands/richtext_format.go @@ -48,8 +48,11 @@ func addRichTextFormatFlag(cmd *cobra.Command) { } // richTextToHTML is the one place a command turns rich-text input into the -// HTML Basecamp stores. With --format html the content is returned exactly as -// given. Otherwise it is read as Markdown, whatever it contains. +// HTML Basecamp stores. With --format html the content is returned as given. +// Otherwise it is read as Markdown, whatever it contains. Either way, a +// mention or file attachment copied from what the API returned is first +// reduced to the bare tag Basecamp expects (richtext.CollapseAttachments): +// sent back expanded, it corrupts the post. // // Content that holds HTML tags was sent as written before Markdown became the // only default, so when --format was left unset and the Markdown keeps raw HTML @@ -61,8 +64,9 @@ func richTextToHTML(cmd *cobra.Command, content string) string { panic(cmd.CommandPath() + " reads rich text but has no --format flag; call addRichTextFormatFlag") } if flag.Value.String() == string(richTextHTML) { - return content + return richtext.CollapseAttachments(content) } + content = richtext.CollapseAttachmentsInMarkdown(content) if !flag.Changed && richtext.HasRichTextHTML(content) { fmt.Fprintln(cmd.ErrOrStderr(), "Warning: content contains HTML tags and was read as Markdown, the default. "+ "Supported tags are kept and the Markdown around them is converted. "+ diff --git a/internal/commands/richtext_format_test.go b/internal/commands/richtext_format_test.go index 849ffbacd..42475b346 100644 --- a/internal/commands/richtext_format_test.go +++ b/internal/commands/richtext_format_test.go @@ -270,3 +270,28 @@ func TestNotesSetFormatHTMLSendsContentExactly(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(transport.last(t).Body), &body)) assert.Equal(t, passthroughHTML, body.Note.Content) } + +// expandedMention is a mention as the API returns it: the avatar figure inside +// the tag and the rendered mention in its content attribute. +const expandedMention = `

Started by

+ Jason Fried +
+ Jason +
+
this week.

` + +// Writing back a mention as the API returned it stored its avatar as content +// of its own, split the paragraph, and added a copy on every edit (#905). +// Basecamp's write form is the bare tag, in either format. +func TestCommentsUpdateCollapsesAMentionCopiedFromTheAPI(t *testing.T) { + want := `

Started by this week.

` + for _, format := range []string{"html", "markdown"} { + t.Run(format, func(t *testing.T) { + transport := &mockCommentWriteTransport{} + _, err := runCommentWithStderr(t, transport, expandedMention+"\n", "update", "1234", "-", "--format", format) + require.NoError(t, err) + assert.Equal(t, want, sentContent(t, transport)) + }) + } +} diff --git a/internal/richtext/attachments.go b/internal/richtext/attachments.go new file mode 100644 index 000000000..31704409a --- /dev/null +++ b/internal/richtext/attachments.go @@ -0,0 +1,180 @@ +package richtext + +import ( + "bytes" + "regexp" + "sort" + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "golang.org/x/net/html" +) + +// reTagAttribute matches one attribute of a tag: its name, and its value if it +// has one, quoted or not. +var reTagAttribute = regexp.MustCompile(`([^\s"'=<>/]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>` + "`" + `]+))?`) + +// CollapseAttachments returns html with every that names a +// record (it has an sgid) reduced to the form Basecamp documents for writing +// one: the tag with nothing inside it and without its content attribute. +// +// Basecamp returns a mention or a file expanded: the person's avatar or the +// file's preview as a
inside the tag, and for a mention its rendered +// HTML in content. Written back as returned, the figure is stored as content +// of its own: it breaks the paragraph around a mention apart, and each edit +// adds another copy. Basecamp expands the bare tag again when it is read, so +// collapsing it loses nothing. An attachment without an sgid carries its +// content itself (a horizontal rule, say) and is left as written, and so is +// one that is self-closing or never closed. +func CollapseAttachments(s string) string { + if !hasAttachmentTag(s) { + return s + } + return collapseAttachments(s, attachmentTagsIn([]byte(s), []text.Segment{text.NewSegment(0, len(s))})) +} + +// CollapseAttachmentsInMarkdown is CollapseAttachments for Markdown. Only an +// attachment tag the Markdown renderer writes as markup counts: one written +// in code, escaped with a backslash, or in an HTML block the renderer shows +// as text is left as written. +func CollapseAttachmentsInMarkdown(md string) string { + if !hasAttachmentTag(md) { + return md + } + return collapseAttachments(md, markdownAttachmentTags(md)) +} + +func hasAttachmentTag(s string) bool { + return strings.Contains(strings.ToLower(s), " tag: where it starts and ends in the +// string it was read from, which kind of tag it is, and the tag as read. A +// tag can span lines whose prefixes (a blockquote's "> ") are not part of it, +// so the tag as read can differ from the source between start and end. +type attachmentTag struct { + start, end int + kind html.TokenType + raw string +} + +// collapseAttachments collapses the attachment elements tags pair up in s. +// An opening tag pairs with the next tag when that tag closes it. One that is +// never closed, or is followed by another attachment first, is left as +// written, and so is a self-closing one. +func collapseAttachments(s string, tags []attachmentTag) string { + var b strings.Builder + last := 0 + for i := 0; i+1 < len(tags); i++ { + open, next := tags[i], tags[i+1] + if open.kind != html.StartTagToken || next.kind != html.EndTagToken { + continue + } + b.WriteString(s[last:open.start]) + b.WriteString(collapseAttachment(s[open.start:next.end], open.raw)) + last = next.end + i++ + } + b.WriteString(s[last:]) + return b.String() +} + +// attachmentTagsIn reads the segments of source, joined, as HTML and +// returns their attachment tags, placed in source. A tokenizer reads them, so +// markup inside a comment or inside another tag's attribute value is not +// taken for a tag. +func attachmentTagsIn(source []byte, segments []text.Segment) []attachmentTag { + var joined []byte + var at []int // at[i] is where joined[i] is in source + for _, seg := range segments { + for range seg.Padding { + joined = append(joined, ' ') + at = append(at, seg.Start) + } + for i := seg.Start; i < seg.Stop; i++ { + joined = append(joined, source[i]) + at = append(at, i) + } + } + + var tags []attachmentTag + z := html.NewTokenizer(bytes.NewReader(joined)) + for pos := 0; ; { + kind := z.Next() + if kind == html.ErrorToken { + return tags + } + size := len(z.Raw()) + isTag := kind == html.StartTagToken || kind == html.EndTagToken || kind == html.SelfClosingTagToken + if name, _ := z.TagName(); isTag && bytes.Equal(name, []byte("bc-attachment")) { + tags = append(tags, attachmentTag{ + start: at[pos], end: at[pos+size-1] + 1, kind: kind, raw: string(joined[pos : pos+size]), + }) + } + pos += size + } +} + +// markdownAttachmentTags returns the attachment tags in md that its renderer +// writes as markup: inline raw HTML, and the HTML blocks it passes through +// (isRichTextTag, as renderHTMLBlock decides), each read from its segments +// as the renderer reads it. Code, backslash escapes and the HTML blocks it +// escapes hold none. +func markdownAttachmentTags(md string) []attachmentTag { + source := []byte(md) + doc := mdConverter.Parser().Parse(text.NewReader(source)) + + var tags []attachmentTag + read := func(segments []text.Segment) { + tags = append(tags, attachmentTagsIn(source, segments)...) + } + _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch node := n.(type) { + case *ast.RawHTML: + read(node.Segments.Sliced(0, node.Segments.Len())) + case *ast.HTMLBlock: + lines := node.Lines() + if isRichTextTag(lines.Value(source)) { + segments := lines.Sliced(0, lines.Len()) + if node.HasClosure() { + segments = append(segments, node.ClosureLine) + } + read(segments) + } + return ast.WalkSkipChildren, nil + } + return ast.WalkContinue, nil + }) + sort.Slice(tags, func(i, j int) bool { return tags[i].start < tags[j].start }) + return tags +} + +// collapseAttachment reduces one element to its write form, +// or returns it unchanged when it names no record. +func collapseAttachment(element, open string) string { + attrs := reTagAttribute.FindAllString(open[len("" +} diff --git a/internal/richtext/attachments_test.go b/internal/richtext/attachments_test.go new file mode 100644 index 000000000..bbbb38438 --- /dev/null +++ b/internal/richtext/attachments_test.go @@ -0,0 +1,160 @@ +package richtext + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCollapseAttachmentsReducesAnExpandedMention(t *testing.T) { + input := `

Hi

+ Jo +
Jo
+
there.

` + + assert.Equal(t, + `

Hi there.

`, + CollapseAttachments(input)) +} + +func TestCollapseAttachmentsKeepsAFileAttachmentsOwnAttributes(t *testing.T) { + input := `
The bug
` + + assert.Equal(t, + ``, + CollapseAttachments(input)) +} + +func TestCollapseAttachmentsLeavesWhatNamesNoRecord(t *testing.T) { + for _, input := range []string{ + `

`, + ``, + ``, + `

no attachments

`, + } { + assert.Equal(t, input, CollapseAttachments(input)) + } +} + +func TestCollapseAttachmentsInMarkdownLeavesCode(t *testing.T) { + expanded := `
Jo
` + bare := `` + input := "Hi " + expanded + " there.\n\n" + + "`" + expanded + "`\n\n" + + "```html\n" + expanded + "\n```\n\n" + + " " + expanded + "\n" + + assert.Equal(t, + "Hi "+bare+" there.\n\n"+ + "`"+expanded+"`\n\n"+ + "```html\n"+expanded+"\n```\n\n"+ + " "+expanded+"\n", + CollapseAttachmentsInMarkdown(input)) +} + +// A self-closing attachment is not the opening tag of the next one: reading it +// that way dropped everything up to the next closing tag. +func TestCollapseAttachmentsSkipsASelfClosingAttachment(t *testing.T) { + input := `KEEP THIS TEXT
Jo
` + + assert.Equal(t, + `KEEP THIS TEXT`, + CollapseAttachments(input)) + assert.Equal(t, + `KEEP THIS TEXT`, + CollapseAttachmentsInMarkdown(input)) +} + +func TestCollapseAttachmentsReadsAQuotedGreaterThan(t *testing.T) { + input := `
a > b
` + + assert.Equal(t, + ``, + CollapseAttachments(input)) +} + +func TestCollapseAttachmentsLeavesAnUnclosedAttachment(t *testing.T) { + for _, input := range []string{ + `

Hi

Jo

`, + `

Hi

Jo
`, + } { + assert.Equal(t, input, CollapseAttachments(input)) + } + + assert.Equal(t, + `KEEP `, + CollapseAttachments(`KEEP
Jo
`)) +} + +// Lowering a string for a case-blind search must not move its bytes: 'İ' +// lowers to three bytes from two. +func TestCollapseAttachmentsAfterTextThatLowersLonger(t *testing.T) { + input := `

İİ

Jo
ok

` + + assert.Equal(t, `

İİ ok

`, CollapseAttachments(input)) +} + +// A backslash makes a tag literal Markdown text, so neither an escaped +// opening tag nor an escaped closing tag is markup. +func TestCollapseAttachmentsInMarkdownLeavesEscapedTags(t *testing.T) { + for _, input := range []string{ + `Example: \KEEP THIS TEXT\`, + `Example: KEEP THIS TEXT\`, + } { + assert.Equal(t, input, CollapseAttachmentsInMarkdown(input)) + } + + assert.Equal(t, + `Example: \\`, + CollapseAttachmentsInMarkdown(`Example: \\
Jo
`)) +} + +// Markup inside an HTML comment is not markup: reading it as an attachment +// deleted the visible content between two comments. +func TestCollapseAttachmentsLeavesTagsInComments(t *testing.T) { + input := `

KEEP THIS TEXT

` + + assert.Equal(t, input, CollapseAttachments(input)) + assert.Equal(t, input, CollapseAttachmentsInMarkdown(input)) + + assert.Equal(t, + `
Jo