refactor (backend): port emoji-regex to backend-rs

Cargo.lock

@@ -209,6 +209,7 @@ dependencies = [
  "cfg-if",
  "chrono",
  "cuid2",
+ "emojis",
  "idna",
  "jsonschema",
  "macro_rs",
@@ -702,6 +703,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "emojis"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ee61eb945bff65ee7d19d157d39c67c33290ff0742907413fd5eefd29edc979"
+dependencies = [
+ "phf",
+]
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.34"
@@ -1662,6 +1672,24 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
+[[package]]
+name = "phf"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.14"
@@ -2313,6 +2341,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
 
+[[package]]
+name = "siphasher"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+
 [[package]]
 name = "slab"
 version = "0.4.9"

Cargo.toml

@@ -17,6 +17,7 @@ cfg-if = "1.0.0"
 chrono = "0.4.37"
 convert_case = "0.6.0"
 cuid2 = "0.1.2"
+emojis = "0.6.1"
 idna = "0.5.0"
 jsonschema = "0.17.1"
 once_cell = "1.19.0"

packages/backend-rs/Cargo.toml

@@ -24,6 +24,7 @@ bcrypt = { workspace = true }
 cfg-if = { workspace = true }
 chrono = { workspace = true }
 cuid2 = { workspace = true }
+emojis = { workspace = true }
 idna = { workspace = true }
 jsonschema = { workspace = true }
 once_cell = { workspace = true }

packages/backend-rs/index.d.ts

@@ -132,6 +132,7 @@ export function isSelfHost(host?: string | undefined | null): boolean
 export function isSameOrigin(uri: string): boolean
 export function extractHost(uri: string): string
 export function toPuny(host: string): string
+export function isUnicodeEmoji(s: string): boolean
 export function sqlLikeEscape(src: string): string
 export function safeForSql(src: string): boolean
 /** Convert milliseconds to a human readable string */

packages/backend-rs/index.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { readServerConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, sqlLikeEscape, safeForSql, formatMilliseconds, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
+const { readServerConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
 
 module.exports.readServerConfig = readServerConfig
 module.exports.stringToAcct = stringToAcct
@@ -321,6 +321,7 @@ module.exports.isSelfHost = isSelfHost
 module.exports.isSameOrigin = isSameOrigin
 module.exports.extractHost = extractHost
 module.exports.toPuny = toPuny
+module.exports.isUnicodeEmoji = isUnicodeEmoji
 module.exports.sqlLikeEscape = sqlLikeEscape
 module.exports.safeForSql = safeForSql
 module.exports.formatMilliseconds = formatMilliseconds

packages/backend-rs/src/misc/emoji.rs

@@ -0,0 +1,31 @@
+#[inline]
+#[crate::export]
+pub fn is_unicode_emoji(s: &str) -> bool {
+    emojis::get(s).is_some()
+}
+
+#[cfg(test)]
+mod unit_test {
+    use super::is_unicode_emoji;
+
+    #[test]
+    fn test_unicode_emoji_check() {
+        assert!(is_unicode_emoji("⭐"));
+        assert!(is_unicode_emoji("👍"));
+        assert!(is_unicode_emoji("❤"));
+        assert!(is_unicode_emoji("♥️"));
+        assert!(is_unicode_emoji("❤️"));
+        assert!(is_unicode_emoji("💙"));
+        assert!(is_unicode_emoji("🩷"));
+        assert!(is_unicode_emoji("🖖🏿"));
+        assert!(is_unicode_emoji("🏃‍➡️"));
+        assert!(is_unicode_emoji("👩‍❤️‍👨"));
+        assert!(is_unicode_emoji("👩‍👦‍👦"));
+        assert!(is_unicode_emoji("🏳️‍🌈"));
+
+        assert!(!is_unicode_emoji("⭐⭐"));
+        assert!(!is_unicode_emoji("x"));
+        assert!(!is_unicode_emoji("\t"));
+        assert!(!is_unicode_emoji(":meow_aww:"));
+    }
+}

packages/backend-rs/src/misc/mod.rs

@@ -1,6 +1,7 @@
 pub mod acct;
 pub mod check_word_mute;
 pub mod convert_host;
+pub mod emoji;
 pub mod escape_sql;
 pub mod format_milliseconds;
 pub mod mastodon_id;

packages/backend/package.json

@@ -33,7 +33,6 @@
 		"@peertube/http-signature": "1.7.0",
 		"@redocly/openapi-core": "1.11.0",
 		"@sinonjs/fake-timers": "11.2.2",
-		"@twemoji/parser": "^15.1.1",
 		"adm-zip": "0.5.10",
 		"ajv": "8.12.0",
 		"archiver": "7.0.1",

packages/backend/src/misc/emoji-regex.ts

@@ -1,5 +0,0 @@
-import twemoji from "@twemoji/parser/dist/lib/regex.js";
-const twemojiRegex = twemoji.default;
-
-export const emojiRegex = new RegExp(`(${twemojiRegex.source})`);
-export const emojiRegexAtStartToEnd = new RegExp(`^(${twemojiRegex.source})$`);

packages/backend/src/misc/reaction-lib.ts

@@ -1,5 +1,4 @@
-import { emojiRegex } from "./emoji-regex.js";
-import { fetchMeta, toPuny } from "backend-rs";
+import { fetchMeta, isUnicodeEmoji, toPuny } from "backend-rs";
 import { Emojis } from "@/models/index.js";
 import { IsNull } from "typeorm";
 
@@ -22,17 +21,15 @@ export async function toDbReaction(
 ): Promise<string> {
 	if (!reaction) return (await fetchMeta(true)).defaultReaction;
 
-	reacterHost = reacterHost == null ? null : toPuny(reacterHost);
-
 	if (reaction.includes("❤") || reaction.includes("♥️")) return "❤️";
 
 	// Allow unicode reactions
-	const match = emojiRegex.exec(reaction);
-	if (match) {
-		const unicode = match[0];
-		return unicode;
+	if (isUnicodeEmoji(reaction)) {
+		return reaction;
 	}
 
+	reacterHost = reacterHost == null ? null : toPuny(reacterHost);
+
 	const custom = reaction.match(/^:([\w+-]+)(?:@\.)?:$/);
 	if (custom) {
 		const name = custom[1];

packages/backend/src/server/api/mastodon/endpoints/status.ts

@@ -1,10 +1,9 @@
 import Router from "@koa/router";
 import { getClient } from "../ApiMastodonCompatibleService.js";
-import { emojiRegexAtStartToEnd } from "@/misc/emoji-regex.js";
 import querystring from "node:querystring";
 import qs from "qs";
 import { convertTimelinesArgsId, limitToInt } from "./timeline.js";
-import { fetchMeta, fromMastodonId } from "backend-rs";
+import { fetchMeta, fromMastodonId, isUnicodeEmoji } from "backend-rs";
 import {
 	convertAccount,
 	convertAttachment,
@@ -37,7 +36,7 @@ export function apiStatusMastodon(router: Router): void {
 			}
 			const text = body.status;
 			const removed = text.replace(/@\S+/g, "").replace(/\s|​/g, "");
-			const isDefaultEmoji = emojiRegexAtStartToEnd.test(removed);
+			const isDefaultEmoji = isUnicodeEmoji(removed);
 			const isCustomEmoji = /^:[a-zA-Z0-9@_]+:$/.test(removed);
 			if ((body.in_reply_to_id && isDefaultEmoji) || isCustomEmoji) {
 				const a = await client.createEmojiReaction(