Refactor: use rust for native mastodon id conversion (#9786)

This uses [napi-rs](https://napi.rs/) to allow for automatic generation of node bindings for the native code.

I also changed the `isolatedModules` TS flag to false to allow for `static enum` to be shared across modules. It doesn't seem to be necessary for the build system that CK uses.

Currently this method does not work with ID generators with longer IDs. Likely the best solution is to add another key in the database.

Some benchmarks for 1 million conversions:

```
	node, x1_000_000: 2.847s
	rust, x1_000_000: 1.265s
```

There are still optimizations that can be made, but I think this is a good starting point and a good way to bring rust into the CK stack.

Co-authored-by: s1idewhist1e <trombonedude05@gmail.com>
Reviewed-on: https://codeberg.org/calckey/calckey/pulls/9786
Co-authored-by: s1idewhist1e <s1idewhist1e@noreply.codeberg.org>
Co-committed-by: s1idewhist1e <s1idewhist1e@noreply.codeberg.org>
This commit is contained in:
s1idewhist1e 2023-03-31 01:58:28 +00:00 committed by Kainoa Kanter
parent 0f9f6ee619
commit 981d3ee725
45 changed files with 692 additions and 34 deletions

View file

@ -91,6 +91,7 @@ If you have access to a server that supports one of the sources below, I recomme
### 🏗️ Build dependencies
- 🦀 [Rust toolchain](https://www.rust-lang.org/)
- 🦬 C/C++ compiler & build tools
- `build-essential` on Debian/Ubuntu Linux
- `base-devel` on Arch Linux

View file

@ -38,6 +38,7 @@
"dependencies": {
"@bull-board/api": "^4.10.2",
"@bull-board/ui": "^4.10.2",
"@napi-rs/cli": "^2.15.0",
"@tensorflow/tfjs": "^3.21.0",
"calckey-js": "^0.0.22",
"js-yaml": "4.1.0",

View file

@ -0,0 +1,3 @@
[target.aarch64-unknown-linux-musl]
linker = "aarch64-linux-musl-gcc"
rustflags = ["-C", "target-feature=-crt-static"]

200
packages/backend/native-utils/.gitignore vendored Normal file
View file

@ -0,0 +1,200 @@
# Created by https://www.toptal.com/developers/gitignore/api/node
# Edit at https://www.toptal.com/developers/gitignore?templates=node
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# Next.js build output
.next
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# End of https://www.toptal.com/developers/gitignore/api/node
# Created by https://www.toptal.com/developers/gitignore/api/macos
# Edit at https://www.toptal.com/developers/gitignore?templates=macos
### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### macOS Patch ###
# iCloud generated files
*.icloud
# End of https://www.toptal.com/developers/gitignore/api/macos
# Created by https://www.toptal.com/developers/gitignore/api/windows
# Edit at https://www.toptal.com/developers/gitignore?templates=windows
### Windows ###
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# End of https://www.toptal.com/developers/gitignore/api/windows
# napi-rs generated files
built/
#Added by cargo
/target
Cargo.lock
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/sdks
!.yarn/versions
*.node

View file

@ -0,0 +1,13 @@
target
Cargo.lock
.cargo
.github
npm
.eslintrc
.prettierignore
rustfmt.toml
yarn.lock
*.node
.yarn
__test__
renovate.json

View file

@ -0,0 +1,18 @@
[package]
edition = "2021"
name = "native-utils"
version = "0.0.0"
[lib]
crate-type = ["cdylib"]
[dependencies]
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
napi = { version = "2.12.0", default-features = false, features = ["napi4"] }
napi-derive = "2.12.0"
[build-dependencies]
napi-build = "2.0.1"
[profile.release]
lto = true

View file

@ -0,0 +1,7 @@
import test from 'ava'
import { sum } from '../index.js'
test('sum from native', (t) => {
t.is(sum(1, 2), 3)
})

View file

@ -0,0 +1,5 @@
extern crate napi_build;
fn main() {
napi_build::setup();
}

View file

@ -0,0 +1,3 @@
# `native-utils-android-arm-eabi`
This is the **armv7-linux-androideabi** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-android-arm-eabi",
"version": "0.0.0",
"os": [
"android"
],
"cpu": [
"arm"
],
"main": "native-utils.android-arm-eabi.node",
"files": [
"native-utils.android-arm-eabi.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-android-arm64`
This is the **aarch64-linux-android** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-android-arm64",
"version": "0.0.0",
"os": [
"android"
],
"cpu": [
"arm64"
],
"main": "native-utils.android-arm64.node",
"files": [
"native-utils.android-arm64.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-darwin-arm64`
This is the **aarch64-apple-darwin** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-darwin-arm64",
"version": "0.0.0",
"os": [
"darwin"
],
"cpu": [
"arm64"
],
"main": "native-utils.darwin-arm64.node",
"files": [
"native-utils.darwin-arm64.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-darwin-universal`
This is the **universal-apple-darwin** binary for `native-utils`

View file

@ -0,0 +1,15 @@
{
"name": "native-utils-darwin-universal",
"version": "0.0.0",
"os": [
"darwin"
],
"main": "native-utils.darwin-universal.node",
"files": [
"native-utils.darwin-universal.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-darwin-x64`
This is the **x86_64-apple-darwin** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-darwin-x64",
"version": "0.0.0",
"os": [
"darwin"
],
"cpu": [
"x64"
],
"main": "native-utils.darwin-x64.node",
"files": [
"native-utils.darwin-x64.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-freebsd-x64`
This is the **x86_64-unknown-freebsd** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-freebsd-x64",
"version": "0.0.0",
"os": [
"freebsd"
],
"cpu": [
"x64"
],
"main": "native-utils.freebsd-x64.node",
"files": [
"native-utils.freebsd-x64.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-linux-arm-gnueabihf`
This is the **armv7-unknown-linux-gnueabihf** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-linux-arm-gnueabihf",
"version": "0.0.0",
"os": [
"linux"
],
"cpu": [
"arm"
],
"main": "native-utils.linux-arm-gnueabihf.node",
"files": [
"native-utils.linux-arm-gnueabihf.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-linux-arm64-gnu`
This is the **aarch64-unknown-linux-gnu** binary for `native-utils`

View file

@ -0,0 +1,21 @@
{
"name": "native-utils-linux-arm64-gnu",
"version": "0.0.0",
"os": [
"linux"
],
"cpu": [
"arm64"
],
"main": "native-utils.linux-arm64-gnu.node",
"files": [
"native-utils.linux-arm64-gnu.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
},
"libc": [
"glibc"
]
}

View file

@ -0,0 +1,3 @@
# `native-utils-linux-arm64-musl`
This is the **aarch64-unknown-linux-musl** binary for `native-utils`

View file

@ -0,0 +1,21 @@
{
"name": "native-utils-linux-arm64-musl",
"version": "0.0.0",
"os": [
"linux"
],
"cpu": [
"arm64"
],
"main": "native-utils.linux-arm64-musl.node",
"files": [
"native-utils.linux-arm64-musl.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
},
"libc": [
"musl"
]
}

View file

@ -0,0 +1,3 @@
# `native-utils-linux-x64-gnu`
This is the **x86_64-unknown-linux-gnu** binary for `native-utils`

View file

@ -0,0 +1,21 @@
{
"name": "native-utils-linux-x64-gnu",
"version": "0.0.0",
"os": [
"linux"
],
"cpu": [
"x64"
],
"main": "native-utils.linux-x64-gnu.node",
"files": [
"native-utils.linux-x64-gnu.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
},
"libc": [
"glibc"
]
}

View file

@ -0,0 +1,3 @@
# `native-utils-linux-x64-musl`
This is the **x86_64-unknown-linux-musl** binary for `native-utils`

View file

@ -0,0 +1,21 @@
{
"name": "native-utils-linux-x64-musl",
"version": "0.0.0",
"os": [
"linux"
],
"cpu": [
"x64"
],
"main": "native-utils.linux-x64-musl.node",
"files": [
"native-utils.linux-x64-musl.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
},
"libc": [
"musl"
]
}

View file

@ -0,0 +1,3 @@
# `native-utils-win32-arm64-msvc`
This is the **aarch64-pc-windows-msvc** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-win32-arm64-msvc",
"version": "0.0.0",
"os": [
"win32"
],
"cpu": [
"arm64"
],
"main": "native-utils.win32-arm64-msvc.node",
"files": [
"native-utils.win32-arm64-msvc.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-win32-ia32-msvc`
This is the **i686-pc-windows-msvc** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-win32-ia32-msvc",
"version": "0.0.0",
"os": [
"win32"
],
"cpu": [
"ia32"
],
"main": "native-utils.win32-ia32-msvc.node",
"files": [
"native-utils.win32-ia32-msvc.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,3 @@
# `native-utils-win32-x64-msvc`
This is the **x86_64-pc-windows-msvc** binary for `native-utils`

View file

@ -0,0 +1,18 @@
{
"name": "native-utils-win32-x64-msvc",
"version": "0.0.0",
"os": [
"win32"
],
"cpu": [
"x64"
],
"main": "native-utils.win32-x64-msvc.node",
"files": [
"native-utils.win32-x64-msvc.node"
],
"license": "MIT",
"engines": {
"node": ">= 10"
}
}

View file

@ -0,0 +1,44 @@
{
"name": "native-utils",
"version": "0.0.0",
"main": "built/index.js",
"types": "built/index.d.ts",
"napi": {
"name": "native-utils",
"triples": {
"additional": [
"aarch64-apple-darwin",
"aarch64-linux-android",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"aarch64-pc-windows-msvc",
"armv7-unknown-linux-gnueabihf",
"x86_64-unknown-linux-musl",
"x86_64-unknown-freebsd",
"i686-pc-windows-msvc",
"armv7-linux-androideabi",
"universal-apple-darwin"
]
}
},
"license": "MIT",
"devDependencies": {
"@napi-rs/cli": "^2.15.0",
"ava": "^5.1.1"
},
"ava": {
"timeout": "3m"
},
"engines": {
"node": ">= 10"
},
"scripts": {
"artifacts": "napi artifacts",
"build": "napi build --platform --release ./built/",
"build:debug": "napi build --platform",
"prepublishOnly": "napi prepublish -t npm",
"test": "ava",
"universal": "napi universal",
"version": "napi version"
}
}

View file

@ -0,0 +1,2 @@
tab_spaces = 2
edition = "2021"

View file

@ -0,0 +1,2 @@
pub mod mastodon_api;

View file

@ -0,0 +1,70 @@
use napi::{bindgen_prelude::*, Error, Status};
use napi_derive::napi;
static CHAR_COLLECTION: &str = "0123456789abcdefghijklmnopqrstuvwxyz";
// -- NAPI exports --
#[napi]
pub enum IdConvertType {
MastodonId,
CalckeyId,
}
#[napi]
pub fn convert_id(in_id: String, id_convert_type: IdConvertType) -> napi::Result<String> {
use IdConvertType::*;
match id_convert_type {
MastodonId => {
let mut out: i64 = 0;
for (i, c) in in_id.to_lowercase().chars().rev().enumerate() {
out += num_from_char(c)? as i64 * 36_i64.pow(i as u32);
}
Ok(out.to_string())
}
CalckeyId => {
let mut input: i64 = match in_id.parse() {
Ok(s) => s,
Err(_) => {
return Err(Error::new(
Status::InvalidArg,
"Unable to parse ID as MasstodonId",
))
}
};
let mut out = String::new();
while input != 0 {
out.insert(0, char_from_num((input % 36) as u8)?);
input /= 36;
}
Ok(out)
}
}
}
// -- end --
#[inline(always)]
fn num_from_char(character: char) -> napi::Result<u8> {
for (i, c) in CHAR_COLLECTION.chars().enumerate() {
if c == character {
return Ok(i as u8);
}
}
Err(Error::new(
Status::InvalidArg,
"Invalid character in parsed base36 id",
))
}
#[inline(always)]
fn char_from_num(number: u8) -> napi::Result<char> {
CHAR_COLLECTION
.chars()
.nth(number as usize)
.ok_or(Error::from_status(Status::Unknown))
}

View file

@ -9,7 +9,7 @@
"migrate": "typeorm migration:run -d ormconfig.js",
"revertmigration": "typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./check_connect.js",
"build": "pnpm swc src -d built -D",
"build": "napi build --platform --release --cargo-cwd native-utils ./native-utils/built/ && pnpm swc src -d built -D",
"watch": "pnpm swc src -d built -D -w",
"lint": "pnpm rome check \"src/**/*.ts\"",
"mocha": "cross-env NODE_ENV=test TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT=\"./test/tsconfig.json\" mocha",
@ -26,6 +26,7 @@
"@bull-board/api": "^4.6.4",
"@bull-board/koa": "^4.6.4",
"@bull-board/ui": "^4.6.4",
"@calckey/megalodon": "5.1.21",
"@discordapp/twemoji": "14.0.2",
"@elastic/elasticsearch": "7.17.0",
"@koa/cors": "3.4.3",
@ -38,12 +39,11 @@
"@tensorflow/tfjs": "^4.2.0",
"ajv": "8.11.2",
"archiver": "5.3.1",
"koa-body": "^6.0.1",
"autobind-decorator": "2.4.0",
"autolinker": "4.0.0",
"axios": "^1.3.2",
"autwh": "0.1.0",
"aws-sdk": "2.1277.0",
"axios": "^1.3.2",
"bcryptjs": "2.4.3",
"blurhash": "1.1.5",
"bull": "4.10.2",
@ -72,12 +72,13 @@
"jsonld": "6.0.0",
"jsrsasign": "10.6.1",
"koa": "2.13.4",
"koa-remove-trailing-slashes": "2.0.3",
"koa-body": "^6.0.1",
"koa-bodyparser": "4.3.0",
"koa-favicon": "2.1.0",
"koa-json-body": "5.3.0",
"koa-logger": "3.2.1",
"koa-mount": "4.0.0",
"koa-remove-trailing-slashes": "2.0.3",
"koa-send": "5.0.1",
"koa-slow": "2.1.0",
"koa-views": "7.0.2",
@ -85,6 +86,7 @@
"mfm-js": "0.23.2",
"mime-types": "2.1.35",
"multer": "1.4.4-lts.1",
"native-utils": "link:native-utils",
"nested-property": "4.0.0",
"node-fetch": "3.3.0",
"nodemailer": "6.8.0",

View file

@ -21,35 +21,10 @@ import discord from "./service/discord.js";
import github from "./service/github.js";
import twitter from "./service/twitter.js";
import { koaBody } from "koa-body";
import { convertId, IdConvertType as IdType } from "native-utils"
export enum IdType {
CalckeyId,
MastodonId
};
export function convertId(idIn: string, idConvertTo: IdType ) {
let idArray = []
switch (idConvertTo) {
case IdType.MastodonId:
idArray = [...idIn].map(item => item.charCodeAt(0));
idArray = idArray.map(item => {
if (item.toString().length < 3) {
return `0${item.toString()}`
}
else return item.toString()
});
return idArray.join('');
case IdType.CalckeyId:
for (let i = 0; i < idIn.length; i += 3) {
if ((idIn.length % 3) !== 0) {
idIn = `0${idIn}`
}
idArray.push(idIn.slice(i, i+3));
}
idArray = idArray.map(item => String.fromCharCode(item));
return idArray.join('');
}
};
// re-export native rust id conversion (function and enum)
export { IdType, convertId };
// Init app
const app = new Koa();

View file

@ -89,7 +89,7 @@ export function apiAccountMastodon(router: Router): void {
}
});
router.get<{ Params: { id: string } }>(
"/v1/accounts/:id(^.*\\d.*$)",
"/v1/accounts/:id",
async (ctx) => {
const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
const accessTokens = ctx.headers.authorization;

View file

@ -21,7 +21,7 @@
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"isolatedModules": false,
"rootDir": "./src",
"baseUrl": "./",
"paths": {

View file

@ -13,6 +13,9 @@ importers:
'@bull-board/ui':
specifier: ^4.10.2
version: 4.10.2
'@napi-rs/cli':
specifier: ^2.15.0
version: 2.15.0
'@tensorflow/tfjs':
specifier: ^3.21.0
version: 3.21.0(seedrandom@3.0.5)
@ -257,6 +260,9 @@ importers:
multer:
specifier: 1.4.4-lts.1
version: 1.4.4-lts.1
native-utils:
specifier: link:native-utils
version: link:native-utils
nested-property:
specifier: 4.0.0
version: 4.0.0
@ -1637,6 +1643,12 @@ packages:
dev: false
optional: true
/@napi-rs/cli@2.15.0:
resolution: {integrity: sha512-RDDr7ZF0cgbd37+NBGeQOjP7Tm/iNM+y3FmrT5bVQBXLePOTuKVC/dBsdN5UZv3Sl2XAwEvBfaGR90E0d8AA6g==}
engines: {node: '>= 10'}
hasBin: true
dev: false
/@nodelib/fs.scandir@2.1.5:
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}