refactor (backend): port emoji-meta to backend-rs

Cargo.toml

@@ -17,6 +17,8 @@ convert_case = "0.6.0"
 cuid2 = "0.1.2"
 emojis = "0.6.1"
 idna = "0.5.0"
+image = "0.25.1"
+nom-exif = "1.2.0"
 once_cell = "1.19.0"
 pretty_assertions = "1.4.0"
 proc-macro2 = "1.0.79"
@@ -24,6 +26,7 @@ quote = "1.0.36"
 rand = "0.8.5"
 redis = "0.25.3"
 regex = "1.10.4"
+reqwest = "0.12.4"
 rmp-serde = "1.2.0"
 sea-orm = "0.12.15"
 serde = "1.0.197"

packages/backend-rs/Cargo.toml

@@ -24,10 +24,13 @@ chrono = { workspace = true }
 cuid2 = { workspace = true }
 emojis = { workspace = true }
 idna = { workspace = true }
+image = { workspace = true }
+nom-exif = { workspace = true }
 once_cell = { workspace = true }
 rand = { workspace = true }
 redis = { workspace = true }
 regex = { workspace = true }
+reqwest = { workspace = true, features = ["blocking"] }
 rmp-serde = { workspace = true }
 sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls"] }
 serde = { workspace = true, features = ["derive"] }

packages/backend-rs/index.d.ts

@@ -248,6 +248,11 @@ export function sqlLikeEscape(src: string): string
 export function safeForSql(src: string): boolean
 /** Convert milliseconds to a human readable string */
 export function formatMilliseconds(milliseconds: number): string
+export interface ImageSize {
+  width: number
+  height: number
+}
+export function getImageSizeFromUrl(url: string): Promise<ImageSize>
 /** TODO: handle name collisions better */
 export interface NoteLikeForGetNoteSummary {
   fileIds: Array<string>

packages/backend-rs/index.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
+const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
 
 module.exports.SECOND = SECOND
 module.exports.MINUTE = MINUTE
@@ -337,6 +337,7 @@ module.exports.isUnicodeEmoji = isUnicodeEmoji
 module.exports.sqlLikeEscape = sqlLikeEscape
 module.exports.safeForSql = safeForSql
 module.exports.formatMilliseconds = formatMilliseconds
+module.exports.getImageSizeFromUrl = getImageSizeFromUrl
 module.exports.getNoteSummary = getNoteSummary
 module.exports.toMastodonId = toMastodonId
 module.exports.fromMastodonId = fromMastodonId

packages/backend-rs/src/misc/get_image_size.rs

@@ -0,0 +1,206 @@
+use crate::misc::redis_cache::{get_cache, set_cache, CacheError};
+use image::{io::Reader, ImageError, ImageFormat};
+use nom_exif::{parse_jpeg_exif, EntryValue, ExifTag};
+use once_cell::sync::OnceCell;
+use std::io::Cursor;
+use std::time::Duration;
+use tokio::sync::Mutex;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Redis cache error: {0}")]
+    CacheErr(#[from] CacheError),
+    #[error("Reqewst error: {0}")]
+    ReqewstErr(#[from] reqwest::Error),
+    #[error("Image decoding error: {0}")]
+    ImageErr(#[from] ImageError),
+    #[error("Image decoding error: {0}")]
+    IoErr(#[from] std::io::Error),
+    #[error("Exif extraction error: {0}")]
+    ExifErr(#[from] nom_exif::Error),
+    #[error("Emoji meta attempt limit exceeded: {0}")]
+    TooManyAttempts(String),
+    #[error("Unsupported image type: {0}")]
+    UnsupportedImageErr(String),
+}
+
+const BROWSER_SAFE_IMAGE_TYPES: [ImageFormat; 8] = [
+    ImageFormat::Png,
+    ImageFormat::Jpeg,
+    ImageFormat::Gif,
+    ImageFormat::WebP,
+    ImageFormat::Tiff,
+    ImageFormat::Bmp,
+    ImageFormat::Ico,
+    ImageFormat::Avif,
+];
+
+static CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
+
+fn client() -> Result<reqwest::Client, reqwest::Error> {
+    CLIENT
+        .get_or_try_init(|| {
+            reqwest::Client::builder()
+                .timeout(Duration::from_secs(5))
+                .build()
+        })
+        .cloned()
+}
+
+static MTX_GUARD: Mutex<()> = Mutex::const_new(());
+
+#[derive(Debug, PartialEq)]
+#[crate::export(object)]
+pub struct ImageSize {
+    pub width: u32,
+    pub height: u32,
+}
+
+#[crate::export]
+pub async fn get_image_size_from_url(url: &str) -> Result<ImageSize, Error> {
+    let attempted: bool;
+
+    {
+        let _ = MTX_GUARD.lock().await;
+
+        let key = format!("fetchImage:{}", url);
+        attempted = get_cache::<bool>(&key)?.is_some();
+
+        if !attempted {
+            set_cache(&key, &true, 10 * 60)?;
+        }
+    }
+
+    if attempted {
+        tracing::warn!("attempt limit exceeded: {}", url);
+        return Err(Error::TooManyAttempts(url.to_string()));
+    }
+
+    tracing::info!("retrieving image size from {}", url);
+
+    let image_bytes = client()?.get(url).send().await?.bytes().await?;
+    let reader = Reader::new(Cursor::new(&image_bytes)).with_guessed_format()?;
+
+    let format = reader.format();
+    if format.is_none() || !BROWSER_SAFE_IMAGE_TYPES.contains(&format.unwrap()) {
+        return Err(Error::UnsupportedImageErr(format!("{:?}", format)));
+    }
+
+    let size = reader.into_dimensions()?;
+
+    let res = ImageSize {
+        width: size.0,
+        height: size.1,
+    };
+
+    if format.unwrap() != ImageFormat::Jpeg {
+        return Ok(res);
+    }
+
+    // handle jpeg orientation
+    // https://magnushoff.com/articles/jpeg-orientation/
+
+    let exif = parse_jpeg_exif(&*image_bytes)?;
+    if exif.is_none() {
+        return Ok(res);
+    }
+
+    let orientation = exif.unwrap().get_value(&ExifTag::Orientation)?;
+    let rotated =
+        orientation.is_some() && matches!(orientation.unwrap(), EntryValue::U32(v) if v >= 5);
+
+    if !rotated {
+        return Ok(res);
+    }
+
+    Ok(ImageSize {
+        width: size.1,
+        height: size.0,
+    })
+}
+
+#[cfg(test)]
+mod unit_test {
+    use super::{get_image_size_from_url, ImageSize};
+    use crate::misc::redis_cache::delete_cache;
+    use pretty_assertions::assert_eq;
+
+    #[tokio::test]
+    async fn test_get_image_size() {
+        let png_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/splash.png";
+        let png_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/notification-badges/at.png";
+        let png_url_3 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/api-doc.png";
+        let rotated_jpeg_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/test/resources/rotate.jpg";
+        let webp_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/custom/assets/badges/error.webp";
+        let webp_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/screenshots/1.webp";
+        let ico_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/favicon.ico";
+        let mp3_url = "https://firefish.dev/firefish/firefish/-/blob/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/sounds/aisha/1.mp3";
+
+        // Delete caches in case you run this test multiple times
+        // (should be disabled in CI tasks)
+        delete_cache(&format!("fetchImage:{}", png_url_1)).unwrap();
+        delete_cache(&format!("fetchImage:{}", png_url_2)).unwrap();
+        delete_cache(&format!("fetchImage:{}", png_url_3)).unwrap();
+        delete_cache(&format!("fetchImage:{}", rotated_jpeg_url)).unwrap();
+        delete_cache(&format!("fetchImage:{}", webp_url_1)).unwrap();
+        delete_cache(&format!("fetchImage:{}", webp_url_2)).unwrap();
+        delete_cache(&format!("fetchImage:{}", ico_url)).unwrap();
+        delete_cache(&format!("fetchImage:{}", mp3_url)).unwrap();
+
+        let png_size_1 = ImageSize {
+            width: 1024,
+            height: 1024,
+        };
+        let png_size_2 = ImageSize {
+            width: 96,
+            height: 96,
+        };
+        let png_size_3 = ImageSize {
+            width: 1024,
+            height: 354,
+        };
+        let rotated_jpeg_size = ImageSize {
+            width: 256,
+            height: 512,
+        };
+        let webp_size_1 = ImageSize {
+            width: 256,
+            height: 256,
+        };
+        let webp_size_2 = ImageSize {
+            width: 1080,
+            height: 2340,
+        };
+        let ico_size = ImageSize {
+            width: 256,
+            height: 256,
+        };
+
+        assert_eq!(
+            png_size_1,
+            get_image_size_from_url(png_url_1).await.unwrap()
+        );
+        assert_eq!(
+            png_size_2,
+            get_image_size_from_url(png_url_2).await.unwrap()
+        );
+        assert_eq!(
+            png_size_3,
+            get_image_size_from_url(png_url_3).await.unwrap()
+        );
+        assert_eq!(
+            rotated_jpeg_size,
+            get_image_size_from_url(rotated_jpeg_url).await.unwrap()
+        );
+        assert_eq!(
+            webp_size_1,
+            get_image_size_from_url(webp_url_1).await.unwrap()
+        );
+        assert_eq!(
+            webp_size_2,
+            get_image_size_from_url(webp_url_2).await.unwrap()
+        );
+        assert_eq!(ico_size, get_image_size_from_url(ico_url).await.unwrap());
+        assert!(get_image_size_from_url(mp3_url).await.is_err());
+    }
+}

packages/backend-rs/src/misc/mod.rs

@@ -6,6 +6,7 @@ pub mod convert_host;
 pub mod emoji;
 pub mod escape_sql;
 pub mod format_milliseconds;
+pub mod get_image_size;
 pub mod get_note_summary;
 pub mod mastodon_id;
 pub mod meta;

packages/backend-rs/src/misc/redis_cache.rs

@@ -3,7 +3,7 @@ use redis::{Commands, RedisError};
 use serde::{Deserialize, Serialize};
 
 #[derive(thiserror::Error, Debug)]
-pub enum Error {
+pub enum CacheError {
     #[error("Redis error: {0}")]
     RedisError(#[from] RedisError),
     #[error("Data serialization error: {0}")]
@@ -12,27 +12,37 @@ pub enum Error {
     DeserializeError(#[from] rmp_serde::decode::Error),
 }
 
+fn prefix_key(key: &str) -> String {
+    redis_key(format!("cache:{}", key))
+}
+
 pub fn set_cache<V: for<'a> Deserialize<'a> + Serialize>(
     key: &str,
     value: &V,
     expire_seconds: u64,
-) -> Result<(), Error> {
+) -> Result<(), CacheError> {
     redis_conn()?.set_ex(
-        redis_key(key),
+        prefix_key(key),
         rmp_serde::encode::to_vec(&value)?,
         expire_seconds,
     )?;
     Ok(())
 }
 
-pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(key: &str) -> Result<Option<V>, Error> {
-    let serialized_value: Option<Vec<u8>> = redis_conn()?.get(redis_key(key))?;
+pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(
+    key: &str,
+) -> Result<Option<V>, CacheError> {
+    let serialized_value: Option<Vec<u8>> = redis_conn()?.get(prefix_key(key))?;
     Ok(match serialized_value {
         Some(v) => Some(rmp_serde::from_slice::<V>(v.as_ref())?),
         None => None,
     })
 }
 
+pub fn delete_cache(key: &str) -> Result<(), CacheError> {
+    Ok(redis_conn()?.del(prefix_key(key))?)
+}
+
 #[cfg(test)]
 mod unit_test {
     use super::{get_cache, set_cache};

packages/backend/src/misc/emoji-meta.ts

@@ -1,59 +0,0 @@
-import probeImageSize from "probe-image-size";
-import { Mutex } from "redis-semaphore";
-
-import { FILE_TYPE_BROWSERSAFE } from "backend-rs";
-import Logger from "@/services/logger.js";
-import { Cache } from "./cache.js";
-import { redisClient } from "@/db/redis.js";
-import { inspect } from "node:util";
-
-export type Size = {
-	width: number;
-	height: number;
-};
-
-const cache = new Cache<boolean>("emojiMeta", 60 * 10); // once every 10 minutes for the same url
-const logger = new Logger("emoji");
-
-export async function getEmojiSize(url: string): Promise<Size> {
-	let attempted = true;
-
-	const lock = new Mutex(redisClient, "getEmojiSize");
-	await lock.acquire();
-
-	try {
-		attempted = (await cache.get(url)) === true;
-		if (!attempted) {
-			await cache.set(url, true);
-		}
-	} finally {
-		await lock.release();
-	}
-
-	if (attempted) {
-		logger.warn(`Attempt limit exceeded: ${url}`);
-		throw new Error("Too many attempts");
-	}
-
-	try {
-		logger.debug(`Retrieving emoji size from ${url}`);
-		const { width, height, mime } = await probeImageSize(url, {
-			timeout: 5000,
-		});
-		if (!(mime.startsWith("image/") && FILE_TYPE_BROWSERSAFE.includes(mime))) {
-			throw new Error("Unsupported image type");
-		}
-		return { width, height };
-	} catch (e) {
-		throw new Error(`Unable to retrieve metadata:\n${inspect(e)}`);
-	}
-}
-
-export function getNormalSize(
-	{ width, height }: Size,
-	orientation?: number,
-): Size {
-	return (orientation || 0) >= 5
-		? { width: height, height: width }
-		: { width, height };
-}

packages/backend/src/queue/processors/system/local-emoji-size.ts

@@ -3,7 +3,7 @@ import { IsNull } from "typeorm";
 import { Emojis } from "@/models/index.js";
 
 import { queueLogger } from "../../logger.js";
-import { getEmojiSize } from "@/misc/emoji-meta.js";
+import { getImageSizeFromUrl } from "backend-rs";
 import { inspect } from "node:util";
 
 const logger = queueLogger.createSubLogger("local-emoji-size");
@@ -21,7 +21,7 @@ export async function setLocalEmojiSizes(
 
 	for (let i = 0; i < emojis.length; i++) {
 		try {
-			const size = await getEmojiSize(emojis[i].publicUrl);
+			const size = await getImageSizeFromUrl(emojis[i].publicUrl);
 			await Emojis.update(emojis[i].id, {
 				width: size.width || null,
 				height: size.height || null,

packages/backend/src/remote/activitypub/models/note.ts

@@ -13,7 +13,7 @@ import { extractPollFromQuestion } from "./question.js";
 import vote from "@/services/note/polls/vote.js";
 import { apLogger } from "../logger.js";
 import type { DriveFile } from "@/models/entities/drive-file.js";
-import { extractHost, isSameOrigin, toPuny } from "backend-rs";
+import { type ImageSize, extractHost, getImageSizeFromUrl, isSameOrigin, toPuny } from "backend-rs";
 import {
 	Emojis,
 	Polls,
@@ -46,7 +46,6 @@ import { UserProfiles } from "@/models/index.js";
 import { In } from "typeorm";
 import { config } from "@/config.js";
 import { truncate } from "@/misc/truncate.js";
-import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
 import { langmap } from "@/misc/langmap.js";
 import { inspect } from "node:util";
 
@@ -488,9 +487,9 @@ export async function extractEmojis(
 					tag.icon!.url !== exists.originalUrl ||
 					!(exists.width && exists.height)
 				) {
-					let size: Size = { width: 0, height: 0 };
+					let size: ImageSize = { width: 0, height: 0 };
 					try {
-						size = await getEmojiSize(tag.icon!.url);
+						size = await getImageSizeFromUrl(tag.icon!.url);
 					} catch {
 						/* skip if any error happens */
 					}
@@ -520,9 +519,9 @@ export async function extractEmojis(
 
 			apLogger.info(`register emoji host=${host}, name=${name}`);
 
-			let size: Size = { width: 0, height: 0 };
+			let size: ImageSize = { width: 0, height: 0 };
 			try {
-				size = await getEmojiSize(tag.icon!.url);
+				size = await getImageSizeFromUrl(tag.icon!.url);
 			} catch {
 				/* skip if any error happens */
 			}

packages/backend/src/server/api/endpoints/admin/emoji/add.ts

@@ -1,12 +1,11 @@
 import define from "@/server/api/define.js";
 import { Emojis, DriveFiles } from "@/models/index.js";
-import { genId } from "backend-rs";
+import { genId, getImageSizeFromUrl } from "backend-rs";
 import { insertModerationLog } from "@/services/insert-moderation-log.js";
 import { ApiError } from "@/server/api/error.js";
 import rndstr from "rndstr";
 import { publishBroadcastStream } from "@/services/stream.js";
 import { db } from "@/db/postgre.js";
-import { getEmojiSize } from "@/misc/emoji-meta.js";
 
 export const meta = {
 	tags: ["admin", "emoji"],
@@ -49,7 +48,7 @@ export default define(meta, paramDef, async (ps, me) => {
 		? file.name.split(".")[0]
 		: `_${rndstr("a-z0-9", 8)}_`;
 
-	const size = await getEmojiSize(file.url);
+	const size = await getImageSizeFromUrl(file.url);
 
 	const emoji = await Emojis.insert({
 		id: genId(),

packages/backend/src/server/api/endpoints/admin/emoji/copy.ts

@@ -1,12 +1,11 @@
 import define from "@/server/api/define.js";
 import { Emojis } from "@/models/index.js";
-import { genId } from "backend-rs";
+import { genId, getImageSizeFromUrl } from "backend-rs";
 import { ApiError } from "@/server/api/error.js";
 import type { DriveFile } from "@/models/entities/drive-file.js";
 import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
 import { publishBroadcastStream } from "@/services/stream.js";
 import { db } from "@/db/postgre.js";
-import { getEmojiSize } from "@/misc/emoji-meta.js";
 
 export const meta = {
 	tags: ["admin", "emoji"],
@@ -76,7 +75,7 @@ export default define(meta, paramDef, async (ps, me) => {
 		throw new ApiError();
 	}
 
-	const size = await getEmojiSize(driveFile.url);
+	const size = await getImageSizeFromUrl(driveFile.url);
 
 	const copied = await Emojis.insert({
 		id: genId(),