Count all URLs in text as 23 characters flat, do not count domain part of usernames (#4427)

* Count all URLs in text as 23 characters flat, do not count domain part of usernames

* Add new status text counting logic to web UI

app/javascript/mastodon/features/compose/components/character_counter.js

@@ -13,12 +13,12 @@ export default class CharacterCounter extends React.PureComponent {
     if (diff < 0) {
       return <span className='character-counter character-counter--over'>{diff}</span>;
     }
+
     return <span className='character-counter'>{diff}</span>;
   }
 
   render () {
     const diff = this.props.max - length(this.props.text);
-
     return this.checkRemainingText(diff);
   }
 

app/javascript/mastodon/features/compose/components/compose_form.js

@@ -18,6 +18,7 @@ import WarningContainer from '../containers/warning_container';
 import { isMobile } from '../../../is_mobile';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { length } from 'stringz';
+import { countableText } from '../util/counter';
 
 const messages = defineMessages({
   placeholder: { id: 'compose_form.placeholder', defaultMessage: 'What is on your mind?' },
@@ -145,9 +146,9 @@ export default class ComposeForm extends ImmutablePureComponent {
   render () {
     const { intl, onPaste, showSearch } = this.props;
     const disabled = this.props.is_submitting;
-    const text = [this.props.spoiler_text, this.props.text].join('');
+    const text     = [this.props.spoiler_text, countableText(this.props.text)].join('');
 
-    let publishText    = '';
+    let publishText = '';
 
     if (this.props.privacy === 'private' || this.props.privacy === 'direct') {
       publishText = <span className='compose-form__publish-private'><i className='fa fa-lock' /> {intl.formatMessage(messages.publish)}</span>;
@@ -203,7 +204,7 @@ export default class ComposeForm extends ImmutablePureComponent {
 
           <div className='compose-form__publish'>
             <div className='character-counter__wrapper'><CharacterCounter max={500} text={text} /></div>
-            <div className='compose-form__publish-button-wrapper'><Button text={publishText} onClick={this.handleSubmit} disabled={disabled || this.props.is_uploading || length(text) > 500 || (text.length !==0 && text.trim().length === 0)} block /></div>
+            <div className='compose-form__publish-button-wrapper'><Button text={publishText} onClick={this.handleSubmit} disabled={disabled || this.props.is_uploading || length(text) > 500 || (text.length !== 0 && text.trim().length === 0)} block /></div>
           </div>
         </div>
       </div>

app/javascript/mastodon/features/compose/util/counter.js

+const urlPlaceholder = 'xxxxxxxxxxxxxxxxxxxxxxx';
+
+export function countableText(inputText) {
+  return inputText
+    .replace(/https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/g, urlPlaceholder)
+    .replace(/(?:^|[^\/\w])@(([a-z0-9_]+)@[a-z0-9\.\-]+)/ig, '@$2');
+};

app/models/account.rb

@@ -44,7 +44,7 @@
 #
 
 class Account < ApplicationRecord
-  MENTION_RE = /(?:^|[^\/[:word:]])@([a-z0-9_]+(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
+  MENTION_RE = /(?:^|[^\/[:word:]])@(([a-z0-9_]+)(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
 
   include AccountAvatar
   include AccountFinderConcern

app/validators/status_length_validator.rb

@@ -5,6 +5,27 @@ class StatusLengthValidator < ActiveModel::Validator
 
   def validate(status)
     return unless status.local? && !status.reblog?
-    status.errors.add(:text, I18n.t('statuses.over_character_limit', max: MAX_CHARS)) if [status.text, status.spoiler_text].join.mb_chars.grapheme_length > MAX_CHARS
+    status.errors.add(:text, I18n.t('statuses.over_character_limit', max: MAX_CHARS)) if too_long?(status)
+  end
+
+  private
+
+  def too_long?(status)
+    countable_length(status) > MAX_CHARS
+  end
+
+  def countable_length(status)
+    total_text(status).mb_chars.grapheme_length
+  end
+
+  def total_text(status)
+    [status.spoiler_text, countable_text(status)].join
+  end
+
+  def countable_text(status)
+    status.text.dup.tap do |new_text|
+      URI.extract(new_text).each { |url| new_text.gsub!(url, 'x' * 23) }
+      new_text.gsub!(Account::MENTION_RE, '@\2')
+    end
   end
 end

spec/models/status_spec.rb

@@ -3,7 +3,7 @@ require 'rails_helper'
 RSpec.describe Status, type: :model do
   let(:alice) { Fabricate(:account, username: 'alice') }
   let(:bob)   { Fabricate(:account, username: 'bob') }
-  let(:other) { Fabricate(:status, account: bob, text: 'Skulls for the skull god! The enemy\'s gates are sideways!')}
+  let(:other) { Fabricate(:status, account: bob, text: 'Skulls for the skull god! The enemy\'s gates are sideways!') }
 
   subject { Fabricate(:status, account: alice) }
 

spec/validators/status_length_validator_spec.rb

+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe StatusLengthValidator do
+  describe '#validate' do
+    it 'does not add errors onto remote statuses'
+    it 'does not add errors onto local reblogs'
+
+    it 'adds an error when content warning is over 500 characters' do
+      status = double(spoiler_text: 'a' * 520, text: '', errors: double(add: nil), local?: true, reblog?: false)
+      subject.validate(status)
+      expect(status.errors).to have_received(:add)
+    end
+
+    it 'adds an error when text is over 500 characters' do
+      status = double(spoiler_text: '', text: 'a' * 520, errors: double(add: nil), local?: true, reblog?: false)
+      subject.validate(status)
+      expect(status.errors).to have_received(:add)
+    end
+
+    it 'adds an error when text and content warning are over 500 characters total' do
+      status = double(spoiler_text: 'a' * 250, text: 'b' * 251, errors: double(add: nil), local?: true, reblog?: false)
+      subject.validate(status)
+      expect(status.errors).to have_received(:add)
+    end
+
+    it 'counts URLs as 23 characters flat' do
+      text   = ('a' * 476) + " http://#{'b' * 30}.com/example"
+      status = double(spoiler_text: '', text: text, errors: double(add: nil), local?: true, reblog?: false)
+
+      subject.validate(status)
+      expect(status.errors).to_not have_received(:add)
+    end
+
+    it 'counts only the front part of remote usernames' do
+      text   = ('a' * 475) + " @alice@#{'b' * 30}.com"
+      status = double(spoiler_text: '', text: text, errors: double(add: nil), local?: true, reblog?: false)
+
+      subject.validate(status)
+      expect(status.errors).to_not have_received(:add)
+    end
+  end
+end