fix/perf (backend): port latest version check to backend-rs, address excessive requests to firefish.dev

docs/api-change.md

@@ -2,6 +2,10 @@
 
 Breaking changes are indicated by the :warning: icon.
 
+## Unreleased
+
+- :warning: Removed `release` endpoint.
+
 ## v20240424
 
 - Added `antennaLimit` field to the response of `meta` and `admin/meta`, and the request of `admin/update-meta` (optional).

packages/backend-rs/index.d.ts

@@ -261,6 +261,7 @@ export interface NoteLikeForGetNoteSummary {
   hasPoll: boolean
 }
 export function getNoteSummary(note: NoteLikeForGetNoteSummary): string
+export function latestVersion(): Promise<string>
 export function toMastodonId(firefishId: string): string | null
 export function fromMastodonId(mastodonId: string): string | null
 export function fetchMeta(useCache: boolean): Promise<Meta>

packages/backend-rs/index.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
+const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
 
 module.exports.SECOND = SECOND
 module.exports.MINUTE = MINUTE
@@ -339,6 +339,7 @@ module.exports.safeForSql = safeForSql
 module.exports.formatMilliseconds = formatMilliseconds
 module.exports.getImageSizeFromUrl = getImageSizeFromUrl
 module.exports.getNoteSummary = getNoteSummary
+module.exports.latestVersion = latestVersion
 module.exports.toMastodonId = toMastodonId
 module.exports.fromMastodonId = fromMastodonId
 module.exports.fetchMeta = fetchMeta

packages/backend-rs/src/misc/latest_version.rs

@@ -0,0 +1,91 @@
+use crate::database::cache;
+use crate::util::http_client::http_client;
+use serde::{Deserialize, Serialize};
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Cache error: {0}")]
+    CacheErr(#[from] cache::Error),
+    #[error("Reqwest error: {0}")]
+    ReqwestErr(#[from] reqwest::Error),
+    #[error("Failed to deserialize JSON: {0}")]
+    JsonErr(#[from] serde_json::Error),
+}
+
+const UPSTREAM_PACKAGE_JSON_URL: &'static str =
+    "https://firefish.dev/firefish/firefish/-/raw/main/package.json";
+
+async fn get_latest_version() -> Result<String, Error> {
+    #[derive(Debug, Deserialize, Serialize)]
+    struct Response {
+        version: String,
+    }
+
+    let res = http_client()?
+        .get(UPSTREAM_PACKAGE_JSON_URL)
+        .send()
+        .await?
+        .text()
+        .await?;
+    let res_parsed: Response = serde_json::from_str(&res)?;
+
+    Ok(res_parsed.version)
+}
+
+#[crate::export]
+pub async fn latest_version() -> Result<String, Error> {
+    let version: Option<String> =
+        cache::get_one(cache::Category::FetchUrl, UPSTREAM_PACKAGE_JSON_URL)?;
+
+    if let Some(v) = version {
+        tracing::trace!("use cached value: {}", v);
+        Ok(v)
+    } else {
+        tracing::trace!("cache is expired, fetching the latest version");
+        let fetched_version = get_latest_version().await?;
+        tracing::trace!("fetched value: {}", fetched_version);
+
+        cache::set_one(
+            cache::Category::FetchUrl,
+            UPSTREAM_PACKAGE_JSON_URL,
+            &fetched_version,
+            3 * 60 * 60,
+        )?;
+        Ok(fetched_version)
+    }
+}
+
+#[cfg(test)]
+mod unit_test {
+    use super::{latest_version, UPSTREAM_PACKAGE_JSON_URL};
+    use crate::database::cache;
+
+    fn validate_version(version: String) {
+        // version: YYYYMMDD
+        assert!(version.len() == 8);
+        assert!(version.chars().all(|c| c.is_ascii_digit()));
+
+        // YYYY
+        assert!(&version[..4] >= "2024");
+
+        // MM
+        assert!(&version[4..6] >= "01");
+        assert!(&version[4..6] <= "12");
+
+        // DD
+        assert!(&version[6..] >= "01");
+        assert!(&version[6..] <= "31");
+    }
+
+    #[tokio::test]
+    async fn check_version() {
+        // TODO: don't need to do this in CI tasks
+        cache::delete_one(cache::Category::FetchUrl, UPSTREAM_PACKAGE_JSON_URL).unwrap();
+
+        // fetch from firefish.dev
+        validate_version(latest_version().await.unwrap());
+
+        // use cache
+        validate_version(latest_version().await.unwrap());
+    }
+}

packages/backend-rs/src/misc/mod.rs

@@ -8,6 +8,7 @@ pub mod escape_sql;
 pub mod format_milliseconds;
 pub mod get_image_size;
 pub mod get_note_summary;
+pub mod latest_version;
 pub mod mastodon_id;
 pub mod meta;
 pub mod nyaify;

packages/backend/src/server/api/endpoints.ts

@@ -286,7 +286,6 @@ import * as ep___pinnedUsers from "./endpoints/pinned-users.js";
 import * as ep___customMotd from "./endpoints/custom-motd.js";
 import * as ep___customSplashIcons from "./endpoints/custom-splash-icons.js";
 import * as ep___latestVersion from "./endpoints/latest-version.js";
-import * as ep___release from "./endpoints/release.js";
 import * as ep___promo_read from "./endpoints/promo/read.js";
 import * as ep___requestResetPassword from "./endpoints/request-reset-password.js";
 import * as ep___resetPassword from "./endpoints/reset-password.js";
@@ -635,7 +634,6 @@ const eps = [
 	["custom-motd", ep___customMotd],
 	["custom-splash-icons", ep___customSplashIcons],
 	["latest-version", ep___latestVersion],
-	["release", ep___release],
 	["promo/read", ep___promo_read],
 	["request-reset-password", ep___requestResetPassword],
 	["reset-password", ep___resetPassword],

packages/backend/src/server/api/endpoints/latest-version.ts

@@ -1,4 +1,5 @@
 import define from "@/server/api/define.js";
+import { latestVersion } from "backend-rs";
 
 export const meta = {
 	tags: ["meta"],
@@ -14,14 +15,7 @@ export const paramDef = {
 } as const;
 
 export default define(meta, paramDef, async () => {
-	let latest_version;
-	await fetch("https://firefish.dev/firefish/firefish/-/raw/main/package.json")
-		.then((response) => response.json())
-		.then((data) => {
-			latest_version = data.version;
-		});
-
 	return {
-		latest_version,
+		latest_version: await latestVersion(),
 	};
 });

packages/backend/src/server/api/endpoints/release.ts

@@ -1,28 +0,0 @@
-import define from "@/server/api/define.js";
-
-export const meta = {
-	tags: ["meta"],
-	description: "Get release notes from Codeberg",
-
-	requireCredential: false,
-	requireCredentialPrivateMode: false,
-} as const;
-
-export const paramDef = {
-	type: "object",
-	properties: {},
-	required: [],
-} as const;
-
-export default define(meta, paramDef, async () => {
-	let release;
-
-	await fetch(
-		"https://firefish.dev/firefish/firefish/-/raw/develop/release.json",
-	)
-		.then((response) => response.json())
-		.then((data) => {
-			release = data;
-		});
-	return release;
-});