From dff0cb799c2fbbef21daa71880e89069a59b68e0 Mon Sep 17 00:00:00 2001 From: naskya Date: Tue, 19 Mar 2024 00:21:02 +0900 Subject: [PATCH] docs: update documents and config files Co-authored-by: sup39 --- .config/example.yml | 8 +- .gitignore | 4 +- CONTRIBUTING.md | 252 +--------- README.md | 75 +-- dev/README.md | 35 -- .../docker-entrypoint.sh | 2 +- .../install.sql | 0 dev/docker-dbonly/README.md | 13 - dev/docker-dbonly/docker-compose.yml | 17 - dev/docker/README.md | 28 -- dev/docker/docker-compose.yml | 52 -- dev/docs/container.md | 30 ++ dev/docs/db-container.md | 98 ++++ dev/docs/local-installation.md | 173 +++++++ dev/manually/README.md | 197 -------- docs/docker.md | 75 --- docs/install-container.md | 47 ++ docs/install.md | 459 +++++++++--------- docs/notice-for-admins.md | 46 +- docs/upgrade.md | 41 ++ scripts/db-down.mjs | 2 +- scripts/db-up.mjs | 2 +- 22 files changed, 691 insertions(+), 965 deletions(-) delete mode 100644 dev/README.md rename dev/{docker => container}/docker-entrypoint.sh (99%) rename dev/{docker-dbonly => db-container}/install.sql (100%) delete mode 100644 dev/docker-dbonly/README.md delete mode 100644 dev/docker-dbonly/docker-compose.yml delete mode 100644 dev/docker/README.md delete mode 100644 dev/docker/docker-compose.yml create mode 100644 dev/docs/container.md create mode 100644 dev/docs/db-container.md create mode 100644 dev/docs/local-installation.md delete mode 100644 dev/manually/README.md delete mode 100644 docs/docker.md create mode 100644 docs/install-container.md create mode 100644 docs/upgrade.md diff --git a/.config/example.yml b/.config/example.yml index 678f9641c9..9082dfb868 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -41,7 +41,7 @@ db: port: 5432 #ssl: false # Database name - db: firefish + db: firefish_db # Auth user: firefish @@ -77,9 +77,9 @@ redis: # A Redis-compatible server (DragonflyDB, Keydb, Redis) for caching # If left blank, it will use the Redis server from above -cacheServer: - host: localhost - port: 6380 +#cacheServer: + #host: localhost + #port: 6379 #family: 0 # 0=Both, 4=IPv4, 6=IPv6 #pass: example-pass #prefix: example-prefix diff --git a/.gitignore b/.gitignore index 291fd025c0..6ed87e24a6 100644 --- a/.gitignore +++ b/.gitignore @@ -60,7 +60,9 @@ packages/backend/assets/LICENSE packages/megalodon/lib packages/megalodon/.idea -dev/docker/firefish +dev/container/firefish +dev/container/db +dev/container/redis # blender backups *.blend1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f414ae0b6..f6e3ff4b02 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,17 @@ Before creating an issue, please check the following: > **Warning** > Do not close issues that are about to be resolved. It should remain open until a commit that actually resolves it is merged. -## Before implementation +## Coding + +### Preparing a development environment + +You can prepare your local Firefish environment in multiple ways: + +- [Run Firefish and databases on your host](../dev/docs/local-installation.md) +- [Run Firefish on your host and databases in containers](../dev/docs/db-container.md) +- [Run Firefish and databases in containers](../dev/docs/container.md) + +### Before implementation When you want to add a feature or fix a bug, **first have the design and policy reviewed in an Issue** (if it is not there, please make one). Without this step, there is a high possibility that the MR will not be merged even if it is implemented. At this point, you also need to clarify the goals of the MR you will create, and make sure that the other members of the team are aware of them. @@ -30,14 +40,14 @@ MRs that do not have a clear set of do's and don'ts tend to be bloated and diffi Also, when you start implementation, assign yourself to the Issue (if you cannot do it yourself, ask another member to assign you). By expressing your intention to work the Issue, you can prevent conflicts in the work. -## Well-known branches +### Well-known branches - The **`main`** branch is tracking the latest release and used for production purposes. - The **`develop`** branch is where we work for the next release. - When you create a MR, basically target it to this branch. **But create a different branch** - The **`l10n_develop`** branch is reserved for localization management. - **`feature/*`** branches are reserved for the development of a specific feature -## Creating a merge request (MR) +### Creating a merge request (MR) Thank you for your MR! Before creating a MR, please check the following: - If possible, prefix the title with a keyword that identifies the type of this MR, as shown below. - `fix` / `refactor` / `feat` / `enhance` / `perf` / `chore` etc. You are also welcome to use gitmoji. This is important as we use these to A) easier read the git history and B) generate our changelog. Without propper prefixing it is possible that your MR is rejected. @@ -68,239 +78,3 @@ Be willing to comment on the good points and not just the things you want fixed - Does the test ensure the expected behavior? - Are there any omissions or gaps? - Does it check for anomalies? - -## More info - -**For more technical details, please visit the Firefish Developers' Corner at [here](dev/README.md) .** - -# THE FOLLOWING IS OUTDATED: - -## Preparing the development environment - -1. Install the following software - - nodejs - - rustup - - cargo - - sea-orm-cli - - podman - - podman-compose -2. Copy the config file - ```sh - cp .config/dev.example.yml .config/default.yml - ``` -3. Start postgres/redis containers - ```sh - pnpm run dev:up - ``` -4. Build Firefish - ```sh - pnpm install - pnpm run build:debug - pnpm run migrate - ``` -5. Start Firefish on your localhost - ```sh - pnpm run start - ``` - -You can use the following commands to initialize the database: -```sh -pnpm run dev:init -pnpm run migrate -``` - -Make sure to clear your browser local storage after initializing the dev instance. - -## Deploy (SOON) -The `/deploy` command by issue comment can be used to deploy the contents of a MR to the preview environment. -``` -/deploy sha= -``` -An actual domain will be assigned so you can test the federation. - -## Merge - -## Release -### Release Instructions -1. Commit version changes in the `develop` branch ([package.json](https://github.com/misskey-dev/misskey/blob/develop/package.json)) -2. Create a release PR. - - Into `master` from `develop` branch. - - The title must be in the format `Release: x.y.z`. - - `x.y.z` is the new version you are trying to release. -3. Deploy and perform a simple QA check. Also verify that the tests passed. -4. Merge it. -5. Create a [release of GitHub](https://github.com/misskey-dev/misskey/releases) - - The target branch must be `master` - - The tag name must be the version - -## Development -During development, it is useful to use the `pnpm run dev` command. -This command monitors the server-side and client-side source files and automatically builds them if they are modified. -In addition, it will also automatically start the Firefish server process. - -## Testing -- Test codes are located in [`/test`](/test). - -### Run test -Create a config file. -``` -cp test/test.yml .config/ -``` -Prepare DB/Redis for testing. -``` -docker-compose -f test/docker-compose.yml up -``` -Alternatively, prepare an empty (data can be erased) DB and edit `.config/test.yml`. - -Run all test. -``` -yarn test -``` - -#### Run specify test -``` -TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT="./test/tsconfig.json" pnpx mocha test/foo.ts --require ts-node/register -``` - -## Vue -Firefish uses Vue(v3) as its front-end framework. -- Use TypeScript. -- **When creating a new component, please use the Composition API (with [setup syntax](https://v3.vuejs.org/api/sfc-script-setup.html) and [ref syntax](https://github.com/vuejs/rfcs/discussions/369)) instead of the Options API.** - - Some of the existing components are implemented in the Options API, but it is an old implementation. Refactors that migrate those components to the Composition API are also welcome. - -## nirax -niraxは、Misskeyで使用しているオリジナルのフロントエンドルーティングシステムです。 -**vue-routerから影響を多大に受けているので、まずはvue-routerについて学ぶことをお勧めします。** - -### ルート定義 -ルート定義は、以下の形式のオブジェクトの配列です。 - -``` ts -{ - name?: string; - path: string; - component: Component; - query?: Record; - loginRequired?: boolean; - hash?: string; - globalCacheKey?: string; - children?: RouteDef[]; -} -``` - -> **Warning** -> 現状、ルートは定義された順に評価されます。 -> たとえば、`/foo/:id`ルート定義の次に`/foo/bar`ルート定義がされていた場合、後者がマッチすることはありません。 - -### 複数のルーター -vue-routerとの最大の違いは、niraxは複数のルーターが存在することを許可している点です。 -これにより、アプリ内ウィンドウでブラウザとは個別にルーティングすることなどが可能になります。 - -## Notes -### How to resolve conflictions occurred at yarn.lock? - -Just execute `yarn` to fix it. - -### INSERTするときにはsaveではなくinsertを使用する -#6441 - -### placeholder -SQLをクエリビルダで組み立てる際、使用するプレースホルダは重複してはならない -例えば -``` ts -query.andWhere(new Brackets(qb => { - for (const type of ps.fileType) { - qb.orWhere(`:type = ANY(note.attachedFileTypes)`, { type: type }); - } -})); -``` -と書くと、ループ中で`type`というプレースホルダが複数回使われてしまいおかしくなる -だから次のようにする必要がある -```ts -query.andWhere(new Brackets(qb => { - for (const type of ps.fileType) { - const i = ps.fileType.indexOf(type); - qb.orWhere(`:type${i} = ANY(note.attachedFileTypes)`, { [`type${i}`]: type }); - } -})); -``` - -### Not `null` in TypeORM -```ts -const foo = await Foos.findOne({ - bar: Not(null) -}); -``` -のようなクエリ(`bar`が`null`ではない)は期待通りに動作しない。 -次のようにします: -```ts -const foo = await Foos.findOne({ - bar: Not(IsNull()) -}); -``` - -### `null` in SQL -SQLを発行する際、パラメータが`null`になる可能性のある場合はSQL文を出し分けなければならない -例えば -``` ts -query.where('file.folderId = :folderId', { folderId: ps.folderId }); -``` -という処理で、`ps.folderId`が`null`だと結果的に`file.folderId = null`のようなクエリが発行されてしまい、これは正しいSQLではないので期待した結果が得られない -だから次のようにする必要がある -``` ts -if (ps.folderId) { - query.where('file.folderId = :folderId', { folderId: ps.folderId }); -} else { - query.where('file.folderId IS NULL'); -} -``` - -### `[]` in SQL -SQLを発行する際、`IN`のパラメータが`[]`(空の配列)になる可能性のある場合はSQL文を出し分けなければならない -例えば -``` ts -const users = await Users.find({ - id: In(userIds) -}); -``` -という処理で、`userIds`が`[]`だと結果的に`user.id IN ()`のようなクエリが発行されてしまい、これは正しいSQLではないので期待した結果が得られない -だから次のようにする必要がある -``` ts -const users = userIds.length > 0 ? await Users.find({ - id: In(userIds) -}) : []; -``` - -### 配列のインデックス in SQL -SQLでは配列のインデックスは**1始まり**。 -`[a, b, c]`の `a`にアクセスしたいなら`[0]`ではなく`[1]`と書く - -### null IN -nullが含まれる可能性のあるカラムにINするときは、そのままだとおかしくなるのでORなどでnullのハンドリングをしよう。 - -### `undefined`にご用心 -MongoDBの時とは違い、findOneでレコードを取得する時に対象レコードが存在しない場合 **`undefined`** が返ってくるので注意。 -MongoDBは`null`で返してきてたので、その感覚で`if (x === null)`とか書くとバグる。代わりに`if (x == null)`と書いてください - -### Migration作成方法 -packages/backendで: -```sh -pnpm dlx typeorm migration:generate -d ormconfig.js -o -``` - -- 生成後、ファイルをmigration下に移してください -- 作成されたスクリプトは不必要な変更を含むため除去してください - -### コネクションには`markRaw`せよ -**Vueのコンポーネントのdataオプションとして**misskey.jsのコネクションを設定するとき、必ず`markRaw`でラップしてください。インスタンスが不必要にリアクティブ化されることで、misskey.js内の処理で不具合が発生するとともに、パフォーマンス上の問題にも繋がる。なお、Composition APIを使う場合はこの限りではない(リアクティブ化はマニュアルなため)。 - -### JSONのimportに気を付けよう -TypeScriptでjsonをimportすると、tscでコンパイルするときにそのjsonファイルも一緒にdistディレクトリに吐き出されてしまう。この挙動により、意図せずファイルの書き換えが発生することがあるので、jsonをimportするときは書き換えられても良いものかどうか確認すること。書き換えされて欲しくない場合は、importで読み込むのではなく、`fs.readFileSync`などの関数を使って読み込むようにすればよい。 - -### コンポーネントのスタイル定義でmarginを持たせない -コンポーネント自身がmarginを設定するのは問題の元となることはよく知られている -marginはそのコンポーネントを使う側が設定する - -## その他 -### HTMLのクラス名で follow という単語は使わない -広告ブロッカーで誤ってブロックされる diff --git a/README.md b/README.md index 06ef89b9fd..67e9f58d4a 100644 --- a/README.md +++ b/README.md @@ -15,75 +15,22 @@ Firefish is based off of Misskey, a powerful microblogging server on ActivityPub
+# Documents + +- [Installation guide](./docs/install.md) +- [Contributing guide](./CONTRIBUTING.md) +- [Changelog](./docs/changelog.md) + # Links -### Want to get involved? Great! - -- If you have the means to, [donations](https://opencollective.com/Firefish) are a great way to keep us going. -- If you know how to program in TypeScript, Vue, or Rust, read the [contributing](./CONTRIBUTING.md) document and our [Developers' Corner](./dev/README.md) . -- If you know a non-English language, translating Firefish on [Weblate](https://hosted.weblate.org/engage/firefish/) help bring Firefish to more people. No technical experience needed! - -### Links - -- Donations: - - OpenCollective: +- Donations: - Matrix space: - Official account: `@firefish@info.firefish.dev` - Weblate: -# Release cycle +# Want to get involved? Great! -We use a rolling release cycle now. Small updates will be frequently merged into the main branch, so please upgrade regularly at your convenience. We will use the commit date (e.g., 20240206) as the version number. +- If you know how to program in TypeScript, Vue, or Rust, please read the [contributing guide](./CONTRIBUTING.md). +- If you have the means to, [donations](https://opencollective.com/Firefish) are a great way to keep us going. +- If you know a non-English language, translating Firefish on [Weblate](https://hosted.weblate.org/engage/firefish/) help bring Firefish to more people. No technical experience needed! -Any special operations required for an upgrade (e.g., installing new dependencies, editing docker-compose.yml) will be noted on [docs/notice-for-admins.md](docs/notice-for-admins.md) , so please check it before upgrading. - -**As always, please take a backup first before starting update.** Whether it is a server snapshot backup or a complete database and file backup, it will be of great help to your recovery process. - -# Getting started - -This guide will work for both **starting from scratch** and **migrating from Misskey**. - - - -## Containerization - -- [How to run Firefish with Docker](https://firefish.dev/firefish/firefish/-/blob/develop/docs/docker.md) -- [How to run Firefish with Kubernetes/Helm](https://firefish.dev/firefish/firefish/-/blob/develop/docs/kubernetes.md) - -## Install Production Environment - -For install production environment details, read [this document](https://firefish.dev/firefish/firefish/-/blob/develop/docs/install.md). - -## Dependencies - -- At least [NodeJS](https://nodejs.org/en/) v18.17.0 (v20/v21 recommended) -- At least [PostgreSQL](https://www.postgresql.org/) v12 (v16 recommended) with [PGroonga](https://pgroonga.github.io/) extension -- At least [Redis](https://redis.io/) v7 -- Web Proxy (one of the following) - - Caddy (recommended for new users) - - Nginx (recommended) - - Apache - -### Optional dependencies - -- [FFmpeg](https://ffmpeg.org/) for video transcoding -- Caching server (one of the following) - - [DragonflyDB](https://www.dragonflydb.io/) (recommended) - - [KeyDB](https://keydb.dev/) - - Another [Redis](https://redis.io/) server - -### Build dependencies - -- At least [Rust](https://www.rust-lang.org/) v1.74 -- C/C++ compiler & build tools - - `build-essential` on Debian/Ubuntu Linux - - `base-devel` on Arch Linux -- [Python 3](https://www.python.org/) - -## Migrating from Misskey/FoundKey to Firefish - -For migrating from Misskey v13, Misskey v12, and FoundKey, read [this document](https://firefish.dev/firefish/firefish/-/blob/develop/docs/migrate.md). diff --git a/dev/README.md b/dev/README.md deleted file mode 100644 index 439db35aca..0000000000 --- a/dev/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Firefish Developers' Corner - -## Contribution guide and code of conduct - -You can also find contribution guide [here](../CONTRIBUTING.md) and code of conduct guide [here](../CODE_OF_CONDUCT.md). - -## Changelog - -You can find more about software change [here](../docs/changelog.md), and API change [here](../docs/api-change.md). - -## API documentation - -You can find interactive API documentation at any Firefish instance /api-doc endpoint. By the way, API console can be found at /api-console endpoint. - -You can also find more about firefish-js [here](../packages/firefish-js/README.md). - -## Firefish's ActivityPub extensions - -You can also find more about Firefish's ActivityPub extensions [here](../docs/activitypub-extensions.md). - -## Environment build - -We support multiple methods to build a development environment, and you can choose the method you like. - -### Containerized Environment - -You can find documentation at [here](docker/README.md). - -### Containerized Database Environment - -You can find documentation at [here](docker-dbonly/README.md). - -### Manually Make Development Environment - -You can find documentation at [here](manually/README.md). diff --git a/dev/docker/docker-entrypoint.sh b/dev/container/docker-entrypoint.sh similarity index 99% rename from dev/docker/docker-entrypoint.sh rename to dev/container/docker-entrypoint.sh index ab82ff2d58..6f60427d66 100755 --- a/dev/docker/docker-entrypoint.sh +++ b/dev/container/docker-entrypoint.sh @@ -18,7 +18,7 @@ if [ ! -f '/.firefish_env_initialized' ]; then # Install compilation dependencies apt-get update - DEBIAN_FRONTEND='noninteractive' apt-get install -y --no-install-recommends build-essential python3 ffmpeg git postgresql-client-16 + DEBIAN_FRONTEND='noninteractive' apt-get install -y --no-install-recommends build-essential python3 ffmpeg git postgresql-client-12 curl -vvv --proto '=https' --tlsv1.2 --show-error --fail https://sh.rustup.rs | sh -s -- -y # Add Cargo PATH diff --git a/dev/docker-dbonly/install.sql b/dev/db-container/install.sql similarity index 100% rename from dev/docker-dbonly/install.sql rename to dev/db-container/install.sql diff --git a/dev/docker-dbonly/README.md b/dev/docker-dbonly/README.md deleted file mode 100644 index 46802e639f..0000000000 --- a/dev/docker-dbonly/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Containerized Database Environment - -The Firefish repo comes with containerized database environment to help make development! - -## Prerequisites - -- Latest [Docker](https://docs.docker.com/get-docker/) installation - - Alternatively, you can use [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose). - -## Start up the environment - -1. Run `git clone https://firefish.dev/firefish/firefish.git && cd firefish` to fetch needed files. -1. Using the scripts located in scripts folder to make the development environment running. diff --git a/dev/docker-dbonly/docker-compose.yml b/dev/docker-dbonly/docker-compose.yml deleted file mode 100644 index d1037fd593..0000000000 --- a/dev/docker-dbonly/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3" - -services: - redis: - image: docker.io/redis:7-alpine - ports: - - "26379:6379" - db: - image: docker.io/groonga/pgroonga:latest-alpine-16-slim - environment: - - "POSTGRES_PASSWORD=password" - - "POSTGRES_USER=firefish" - - "POSTGRES_DB=firefish_db" - ports: - - "25432:5432" - volumes: - - "./install.sql:/docker-entrypoint-initdb.d/install.sql:ro" diff --git a/dev/docker/README.md b/dev/docker/README.md deleted file mode 100644 index 4a7f15822d..0000000000 --- a/dev/docker/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Containerized Environment - -The Firefish repo comes with a new containerized environment to help make development! - -## Prerequisites - -- Latest [Docker](https://docs.docker.com/get-docker/) installation - - Alternatively, you can use [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose). -- The following ports are not in use - - 3030 - - 25432 - - 26379 - -## Start up the environment - -1. Download the [`dev/docker` directory](https://firefish.dev/firefish/firefish/-/tree/develop/dev/docker) and execute `chmod +x docker-entrypoint.sh`. - - Alternatively, you can manually run `git clone https://firefish.dev/firefish/firefish.git && cd firefish` to fetch needed files, or let the script take care of itself. -1. Open `docker-compose.yml` and set `URL` to the URL you want to use (or leave it to `http://localhost:3030`). -1. Run `docker compose up`. This will build the environment, dependencies and prepare the needed config files. - - If you use Podman, you should run `podman-compose up` instead. -1. Wait until the following message shows up. - ```log - DONE * [core boot] All workers started - DONE * [core boot] Now listening on port 3030 on https://your_firefish_url.example.com - ``` -1. A fresh Firefish environment is created on the URL you have set! - -When you want to restart the dev server, you just need to terminate the process (a.k.a. press `Ctrl+C`) and run `docker compose up` again. diff --git a/dev/docker/docker-compose.yml b/dev/docker/docker-compose.yml deleted file mode 100644 index 1a255caa7a..0000000000 --- a/dev/docker/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: "3" - -services: - web: - image: docker.io/node:21-slim - container_name: firefish_web - restart: unless-stopped - depends_on: - - db - - redis - ports: - - "3030:3030" - networks: - - firefishnet - environment: - # Currently bug in the development env and it cannot run. - NODE_ENV: production - PGPASSWORD: password - URL: http://localhost:3030 - volumes: - - ./firefish:/firefish:rw - - ./docker-entrypoint.sh:/docker-entrypoint.sh:rw - entrypoint: /docker-entrypoint.sh - - redis: - restart: unless-stopped - image: docker.io/redis:7.0-alpine - container_name: firefish_redis - networks: - - firefishnet - ports: - - "26379:6379" - volumes: - - ./redis:/data - - db: - restart: unless-stopped - image: docker.io/groonga/pgroonga:latest-alpine-16-slim - container_name: firefish_db - networks: - - firefishnet - environment: - - "POSTGRES_PASSWORD=password" - - "POSTGRES_USER=firefish" - - "POSTGRES_DB=firefish_db" - ports: - - "25432:5432" - volumes: - - ./db:/var/lib/postgresql/data - -networks: - firefishnet: diff --git a/dev/docs/container.md b/dev/docs/container.md new file mode 100644 index 0000000000..e1a570c0d8 --- /dev/null +++ b/dev/docs/container.md @@ -0,0 +1,30 @@ +# Set up a fully-containerized development environment + +## Prerequisites + +- Container runtime installation + - [Docker](https://docs.docker.com/get-docker/) + - [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose) + - [containerd](https://github.com/containerd/containerd) and [nerdctl](https://github.com/containerd/nerdctl) + - or whatever you want to use +- The following ports are not in use + - 3030 + - 25432 + - 26379 + +## Start up the environment + +1. Download the [`dev/container` directory](./dev/container) and execute `chmod +x docker-entrypoint.sh` + - Alternatively, you can use `git clone https://firefish.dev/firefish/firefish.git && cd firefish/dev/container`, although this will clone the entire repository. +1. Edit `docker-compose.yml` and set `URL` to the URL you want to use (or leave it as `http://localhost:3030`) +1. Run `docker compose up` + - This will build the environment, install dependencies and prepare the needed config files. + - If you use Podman, you should run `podman-compose up` instead. +1. Wait until the following message shows up + ```log + DONE * [core boot] All workers started + DONE * [core boot] Now listening on port 3030 on https://your_firefish_url.example.com + ``` +1. A fresh Firefish environment is created on the URL you have set! + +When you want to restart the dev server, you just need to terminate the process (a.k.a. press `Ctrl+C`) and run `docker compose up` again. diff --git a/dev/docs/db-container.md b/dev/docs/db-container.md new file mode 100644 index 0000000000..1b323fde4a --- /dev/null +++ b/dev/docs/db-container.md @@ -0,0 +1,98 @@ +# Set up database containers and run Firefish locally + +## Prerequisites + +- Dependencies + - git + - Node.js + - pnpm + - Rust toolchain + - FFmpeg + - Container runtime + - [Docker](https://docs.docker.com/get-docker/) + - [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose) + - [containerd](https://github.com/containerd/containerd) and [nerdctl](https://github.com/containerd/nerdctl) + - or whatever you want to use +- The following ports are not in use + - 25432 + - 26379 + +You can refer to [local-installation.md](./local-installation.md) to install the dependencies. + +## Configure the environment + +1. Fork the Firefish repository on GitLab +1. Clone your Firefish repository + ```sh + git clone https://firefish.dev/your-user-name/firefish.git + cd firefish + ``` +1. Create `.config/default.yml` with the following content + ```yaml + # You can change the port if 3000 is already used + url: http://localhost:3000 + port: 3000 + + db: + host: localhost + port: 25432 + db: firefish_db + user: firefish + pass: password + + redis: + host: localhost + port: 26379 + + logLevel: [ + 'error', + 'success', + 'warning', + 'info' + ] + ``` +1. Start database containers + ```sh + cd dev/db-container + docker compose up --detach + # or podman-compose up --detach + # or whatever + + # go back to the repository root + cd ../.. + ``` + +## Build and start Firefish + +1. Build Firefish + ```sh + pnpm install + pnpm run build:debug + ``` +1. Execute database migrations + ```sh + pnpm run migrate + ``` +1. Start Firefish + ```sh + pnpm run start + ``` + You can access to the local Firefish server on http://localhost:3000 after this message shows up! + ``` + DONE * [core boot] All workers started + DONE * [core boot] Now listening on port 3000 on http://localhost:3000 + ``` + +## Reset the environment + +You can recreate a fresh local Firefish environment by recreating the database containers: + +```sh +cd dev/db-container +docker compose down +docker compose up --detach +cd ../.. + +pnpm run migrate +pnpm run start +``` diff --git a/dev/docs/local-installation.md b/dev/docs/local-installation.md new file mode 100644 index 0000000000..a13aa6437f --- /dev/null +++ b/dev/docs/local-installation.md @@ -0,0 +1,173 @@ +# Set up a development environment by installing all dependencies locally + +This document demonstrates an example procedure to set up a Firefish development environment on Debian 12. You can refer to this document if you prefer to install all dependencies (Node.js, PostgreSQL, Redis, etc.) locally. + +Make sure that you can use the `sudo` command before proceeding. + +## 1. Install dependencies + +### Utilities + +```sh +sudo apt update +sudo apt install build-essential python3 curl wget git lsb-release +``` + +### Node.js + +Firefish requires Node.js v18.17.0 or later. While you can choose any versions between v18.17.0 and the latest version (v21.6.2 as of writing), we recommend that you install v18.x so as not to use new features inadvertently and introduce incompatibility issues. + +Instructions can be found at [this repository](https://github.com/nodesource/distributions). + +```sh +NODE_MAJOR=18 +curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | sudo -E bash - +sudo apt install nodejs + +# check version +node --version +``` + +### Rust toolchain + +Instructions can be found at [this page](https://www.rust-lang.org/tools/install). + +```sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +. "${HOME}/.cargo/env" + +# check version +cargo --version +``` + +### PostgreSQL and PGroonga + +PostgreSQL install instructions can be found at [this page](https://www.postgresql.org/download/). + +```sh +sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +sudo apt update +sudo apt install postgresql-12 + +sudo systemctl enable --now postgresql + +# check version +psql --version +``` + +PGroonga install instructions can be found at [this page](https://pgroonga.github.io/install/). + +```sh +wget "https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" +sudo apt install "./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" +wget "https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release --codename --short).deb" +sudo apt install "./groonga-apt-source-latest-$(lsb_release --codename --short).deb" +sudo apt update +sudo apt install postgresql-12-pgdg-pgroonga + +rm "apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" "groonga-apt-source-latest-$(lsb_release --codename --short).deb" +``` + +### Redis + +Instructions can be found at [this page](https://redis.io/docs/install/install-redis/). + +```sh +curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list +sudo apt update +sudo apt install redis + +sudo systemctl enable --now redis-server + +# check version +redis-cli --version +``` + +### FFmpeg + +```sh +sudo apt install ffmpeg +``` + +## 2. Set up a database + +1. Create a database user + ```sh + sudo -u postgres createuser --no-createdb --no-createrole --no-superuser --encrypted --pwprompt firefish + ``` + If you forgot the password you typed, you can reset it by executing `sudo -u postgres psql -c "ALTER USER firefish PASSWORD 'password';"`. +2. Create a database + ```sh + sudo -u postgres createdb --encoding='UTF8' --owner=firefish firefish_db + ``` +3. Enable PGronnga extension + ```sh + sudo -u postgres psql --command='CREATE EXTENSION pgroonga;' --dbname=firefish_db + ``` + +## 3. Configure Firefish + +1. Fork the Firefish repository on GitLab +1. Clone your Firefish repository + ```sh + git clone https://firefish.dev/your-user-name/firefish.git + ``` +1. Create the config file + ```sh + cd firefish + vim .config/default.yml + ``` + + ```yaml + url: http://localhost:3000 + port: 3000 + + db: + host: localhost + port: 5432 + db: firefish_db + user: firefish + pass: password + + redis: + host: localhost + port: 6379 + + logLevel: [ + 'error', + 'success', + 'warning', + 'info' + ] + ``` + +## 4. Build and start Firefish + +1. Install pnpm + ```sh + sudo corepack enable + corepack prepare pnpm@latest --activate + + # check version + pnpm --version + ``` +1. Build + ```sh + pnpm install + pnpm run build:debug + ``` +1. Execute database migrations + ```sh + pnpm run migrate + ``` +1. Start Firefish + ```sh + pnpm run start + ``` + You can access to the local Firefish server on http://localhost:3000 after this message shows up! + ``` + DONE * [core boot] All workers started + DONE * [core boot] Now listening on port 3000 on http://localhost:3000 + ``` diff --git a/dev/manually/README.md b/dev/manually/README.md deleted file mode 100644 index b454b9de1d..0000000000 --- a/dev/manually/README.md +++ /dev/null @@ -1,197 +0,0 @@ -# Manually Make Development Environment - -For scenarios with special needs, you may want to manually create a development environment. Hope this article can help you. - -## Introduction - -This article is written based on the Debian Bookworm. Other systems can refer to this article for deployment. However, it is recommended that new users use the same system or Docker environment as ours to avoid wasting time on environment configuration issues. - -The versions of Node.js, Rust, PostgreSQL that come with Debian Bookworm are low, the latest official versions of these components are used to install them. Other components are installed using the apt package manager that comes with the system. - -## Allow `sudo` command - -```sh -su - -apt install -y -V sudo -# user is your username -usermod -aG sudo user -reboot -``` - -## Install Base Requirements - -```sh -sudo apt update -sudo apt install -y -V wget curl git ca-certificates lsb-release gnupg -``` - -## Install Node.js - -The latest version at the time of writing is v21.6.2. Please replace it with the latest Node.js version number during installation. Details can be found in [nodejs.org](https://nodejs.org) . - -1. Download and extract. - -```sh -VERSION=v21.6.2 -DISTRO=linux-x64 -sudo mkdir -p /usr/local/lib/nodejs -wget https://nodejs.org/dist/v21.6.2/node-$VERSION-$DISTRO.tar.xz -sudo tar -xJvf node-$VERSION-$DISTRO.tar.xz -C /usr/local/lib/nodejs -``` - -2. Open your `.profile` and `/root/.profile` files. - -```sh -nano ~/.profile -sudo nano /root/.profile -``` - -3. Add below content at below of this two file to set the environment variable. - -```sh -# Nodejs -VERSION=v21.6.2 -DISTRO=linux-x64 -export PATH=/usr/local/lib/nodejs/node-$VERSION-$DISTRO/bin:$PATH -``` - -4. Refresh `PATH` and test. - -```sh -. ~/.profile -node -v -# Switching to root -sudo -i -. ~/.profile -node -v -exit -``` - -## Install Rust - -1. Running this script and choose "Proceed with installation" option. - -```sh -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -. ~/.profile -cargo -V -``` - -## Install PostgreSQL with PGroonga extension - -```sh -wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -wget https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release --codename --short).deb -echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release --codename --short)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - -sudo apt update -sudo apt install -y -V postgresql-16-pgdg-pgroonga -``` - -## Configuration PostgreSQL - -Create Firefish database, user and PGroonga extension. - -Please not forget prompt database password in console. - -```sh -sudo --user=postgres createuser --no-createdb --no-createrole --no-superuser --encrypted --pwprompt firefish -sudo --user=postgres createdb --encoding='UTF8' --owner=firefish firefish_db -sudo --user=postgres psql --command='CREATE EXTENSION pgroonga;' --dbname=firefish_db -``` - -If you forgot it, run `sudo -u postgres psql -c "ALTER USER firefish PASSWORD 'password';"` to set a password for firefish. - -## Install Redis, Python 3 and build-essential - -```sh -sudo apt update -sudo apt install -y -V redis python3 build-essential -``` - -## Install optional ffmpeg dependencies - -```sh -sudo apt update -sudo apt install -y -V ffmpeg -``` - -## Download and configuration Firefish - -1. Download Firefish and Copy example configuration file. - -```sh -# cd /path/to/your/firefish -git clone https://firefish.dev/firefish/firefish.git -cd firefish/ -cp .config/devenv.yml .config/default.yml -sed -i "s/host: firefish_db/host: localhost/" .config/default.yml -sed -i "s/host: firefish_redis/host: localhost/" .config/default.yml -``` - -2. Open your `default.yml` files and make changes like `URL`, `db/host` `redis/host`. - -```sh -nano .config/default.yml -``` - -## Install package - -1. Let corepack enable. - -```sh -# Switching to root -sudo -i -# cd /path/to/your/firefish -cd /home/user/firefish -npm i -g pm2 -corepack enable -exit -``` - -2. Install dependency. - -```sh -corepack prepare pnpm@latest --activate -pm2 install pm2-logrotate -pnpm install --frozen-lockfile --prod false -``` - -> **Note** -> [`pm2-logrotate`](https://github.com/keymetrics/pm2-logrotate/blob/master/README.md) ensures that log files don't infinitely gather size, as Firefish produces a lot of logs. - -## Start - -1. Build and migrate - -```sh -pnpm install --prod false -NODE_ENV=production -pnpm run build:debug -pnpm run migrate -``` - -2. Start Firefish - -```sh -pnpm run start -``` - -2. Wait until the following message shows up. - -```log -DONE * [core boot] All workers started -DONE * [core boot] Now listening on port 3030 on https://your_firefish_url.example.com -``` - -3. A fresh Firefish environment is created on the URL you have set! - -4. If you want Firefish to run in the background, start it with this command. - -```sh -pm2 start "NODE_ENV=production pnpm run start" --name Firefish -# When you want display log -pm2 logs Firefish -``` diff --git a/docs/docker.md b/docs/docker.md deleted file mode 100644 index d8469ab906..0000000000 --- a/docs/docker.md +++ /dev/null @@ -1,75 +0,0 @@ -# Running a Firefish server with Docker - -## Prerequisites - -- Latest [Docker](https://docs.docker.com/get-docker/) installation - - Alternatively, you can use [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose). -- The following ports are not in use - - 80 - - 443 - - 3000 - - 5432 - - 6379 - -## Pre-built docker container - -[registry.firefish.dev/firefish/firefish](https://firefish.dev/firefish/firefish/container_registry) - -## Config files - -There are example config files that you can use to build the container from source - -- docker-compose.example.yml (**compose file**) -- .config/docker_example.env (**db config settings**) -- .config/default.yml (**Firefish server settings**) - -## Configuring - -Copy the files: - -```sh -cp docker-compose.example.yml docker-compose.yml -cp .config/example.yml .config/default.yml -cp .config/docker_example.env .config/docker.env -``` - -then edit them according to your environment. -You can configure `docker.env` with anything you like, but you will have to pay attention to the `default.yml` file: - -- `url` should be set to the URL you will be hosting the web interface for the server at. -- `host`, `db`, `user`, `pass` will have to be configured in the `PostgreSQL configuration` section - `host` is the name of the postgres container (eg: *firefish_db_1*), and the others should match your `docker.env`. -- `host`will need to be configured in the *Redis configuration* section - it is the name of the redis container (eg: *firefish_redis_1*) - -Everything else can be left as-is. - -## Running - -The [prebuilt container for firefish](https://firefish.dev/firefish/firefish/container_registry) is fairly large, and may take a few minutes to download and extract using docker. - -Copy `docker-compose.yml` and the `config/` to a directory, then run the **docker compose** command: `docker compose up -d` . - - If you use Podman, you should run `podman-compose up` instead. - -NOTE: This will take some time to come fully online, even after download and extracting the container images, and it may emit some error messages before completing successfully. Specifically, the `db` container needs to initialize and so isn't available to the `web` container right away. Only once the `db` container comes online does the `web` container start building and initializing the firefish tables. - -Once the server is up you can use a web browser to access the web interface at `http://serverip:3000` (where `serverip` is the IP of the server you are running the firefish server on). - -## Install reverse proxy - -### Caddy (recommended for new users) - -- Add the following block to your `Caddyfile`, replacing `example.tld` with your own domain: - -```caddy -example.tld { - reverse_proxy http://127.0.0.1:3000 -} -``` - -- Reload your caddy configuration - -### Nginx (recommended) - -- Run `sudo cp ./firefish.nginx.conf /etc/nginx/sites-available/ && cd /etc/nginx/sites-available/` -- Edit `firefish.nginx.conf` to reflect your server properly -- Run `sudo ln -s ./firefish.nginx.conf ../sites-enabled/firefish.nginx.conf` -- Run `sudo nginx -t` to validate that the config is valid, then restart the NGINX service. diff --git a/docs/install-container.md b/docs/install-container.md new file mode 100644 index 0000000000..6b304f772e --- /dev/null +++ b/docs/install-container.md @@ -0,0 +1,47 @@ +# Running a Firefish server with containers + +## Prerequisites + +- Latest [Docker](https://docs.docker.com/get-docker/) installation + - You can also use [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose). + +## Configuration + +Copy the example config files: + +```sh +cp docker-compose.example.yml docker-compose.yml +cp .config/example.yml .config/default.yml +cp .config/docker_example.env .config/docker.env +``` + +then edit them according to your environment. +You can configure `docker.env` with anything you like, but you will have to pay attention to the `default.yml` file: + +- `url` should be set to the URL you will be hosting the web interface for the server at. +- `host`, `db`, `user`, `pass` will have to be configured in the `PostgreSQL configuration` section - `host` is the name of the postgres container (eg: *firefish_db_1*), and the others should match your `docker.env`. +- `host`will need to be configured in the *Redis configuration* section - it is the name of the redis container (eg: *firefish_redis_1*) + +Everything else can be left as-is. + +## Pull the container image + +The image tag is [`registry.firefish.dev/firefish/firefish:latest`](https://firefish.dev/firefish/firefish/container_registry). + +```sh +docker pull registry.firefish.dev/firefish/firefish:latest +# or podman pull registry.firefish.dev/firefish/firefish:latest +``` + +## Run + +```sh +docker compose up --detach +# or podman-compose up --detach +``` + +NOTE: This will take some time to come fully online, even after download and extracting the container images, and it may emit some error messages before completing successfully. Specifically, the `db` container needs to initialize and so isn't available to the `web` container right away. Only once the `db` container comes online does the `web` container start building and initializing the firefish tables. + +Once the server is up you can use a web browser to access the web interface at `http://serverip:3000` (where `serverip` is the IP of the server you are running the firefish server on). + +To publish your server, please follow the instructions in [section 5 of this installation guide](./install.md#5-preparation-for-publishing-a-server). diff --git a/docs/install.md b/docs/install.md index 5e6fba3a43..cab441db1f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,256 +1,292 @@ -# Install Production Environment +# Install Firefish -Hope this article can help you about install production environment. +This document shows an example procedure for installing Firefish on Debian 12. Note that there is much room for customizing the server setup; this document merely demonstrates a simple installation. -## Introduction +If you want to use the pre-built container image, please refer to [`install-container.md`](./install-container.md). -This article is written based on the Debian Bookworm. Other systems can refer to this article for deployment. However, it is recommended that new users use the same system or Docker environment as ours to avoid wasting time on environment configuration issues. +Make sure that you can use the `sudo` command before proceeding. -The versions of Node.js, Rust, PostgreSQL, DragonflyDB that come with Debian Bookworm are low or not have, the latest official versions of these components are used to install them. Other components are installed using the apt package manager that comes with the system. +## 1. Install dependencies -## Allow `sudo` command - -```sh -su - -apt install -y -V sudo -# user is your username -usermod -aG sudo user -reboot -``` - -## Install Base Requirements +### Utilities ```sh sudo apt update -sudo apt install -y -V wget curl git ca-certificates lsb-release gnupg +sudo apt install build-essential python3 curl wget git lsb-release ``` -## Install Node.js +### Node.js and pnpm -The latest version at the time of writing is v21.6.2. Please replace it with the latest Node.js version number during installation. Details can be found in [nodejs.org](https://nodejs.org) . - -1. Download and extract. +Instructions can be found at [this repository](https://github.com/nodesource/distributions). ```sh -VERSION=v21.6.2 -DISTRO=linux-x64 -sudo mkdir -p /usr/local/lib/nodejs -wget https://nodejs.org/dist/v21.6.2/node-$VERSION-$DISTRO.tar.xz -sudo tar -xJvf node-$VERSION-$DISTRO.tar.xz -C /usr/local/lib/nodejs +NODE_MAJOR=20 +curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | sudo -E bash - +sudo apt install nodejs + +# check version +node --version ``` -2. Open your `.profile` and `/root/.profile` files. - +You also need to enable `pnpm`. ```sh -nano ~/.profile -sudo nano /root/.profile +sudo corepack enable +corepack prepare pnpm@latest --activate + +# check version +pnpm --version ``` -3. Add below content at below of this two file to set the environment variable. +### PostgreSQL and PGroonga + +PostgreSQL install instructions can be found at [this page](https://www.postgresql.org/download/). ```sh -# Nodejs -VERSION=v21.6.2 -DISTRO=linux-x64 -export PATH=/usr/local/lib/nodejs/node-$VERSION-$DISTRO/bin:$PATH -``` - -4. Refresh `PATH` and test. - -```sh -. ~/.profile -node -v -# Switching to root -sudo -i -. ~/.profile -node -v -exit -``` - -## Install Rust - -1. Running this script and choose "Proceed with installation" option. - -```sh -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -. ~/.profile -cargo -V -``` - -## Install PostgreSQL with PGroonga extension - -```sh -wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -wget https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release --codename --short).deb -echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release --codename --short)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list +sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt update -sudo apt install -y -V postgresql-16-pgdg-pgroonga +sudo apt install postgresql-16 + +sudo systemctl enable --now postgresql + +# check version +psql --version ``` -## Configuration PostgreSQL - -1. Execute this to running `psql` as `postgres` user. - -```sh -sudo -u postgres psql -``` - -2. Create Firefish database, user and PGroonga extension. **Please change the password.** - -```sql -CREATE DATABASE firefish WITH ENCODING = 'UTF8'; -\connect firefish -CREATE EXTENSION pgroonga; -CREATE USER firefish WITH PASSWORD 'password'; -ALTER USER firefish WITH SUPERUSER; -GRANT ALL ON DATABASE firefish TO firefish; -``` - -3. Run `exit` to return. - -## Install DragonflyDB (Cache) - -```sh -wget https://dragonflydb.gateway.scarf.sh/latest/dragonfly_amd64.deb -sudo dpkg -i dragonfly_amd64.deb -``` - -## Configuration DragonflyDB - -If you experience a lot of traffic, it's a good idea to set up another Redis-compatible caching server. If you don't set one one up, it'll fall back to the mandatory Redis server. DragonflyDB is the recommended option due to its unrivaled performance and ease of use. - -1. Open your `dragonfly.conf` files. - -```sh -sudo nano /etc/dragonfly/dragonfly.conf -``` - -2. Add content at below of this file to set the different port variable because default port is 6379. - -```conf ---port=6380 -``` - -3. Run `sudo systemctl restart dragonfly` to restart it. - -## Install Caddy, Redis, Python 3 and build-essential - -If you already have experience using nginx, you can consider replacing caddy with nginx here. +PGroonga install instructions can be found at [this page](https://pgroonga.github.io/install/). ```sh +wget "https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" +sudo apt install "./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" +wget "https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release --codename --short).deb" +sudo apt install "./groonga-apt-source-latest-$(lsb_release --codename --short).deb" sudo apt update -sudo apt install -y -V caddy redis python3 build-essential +sudo apt install postgresql-16-pgdg-pgroonga + +rm "apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb" "groonga-apt-source-latest-$(lsb_release --codename --short).deb" ``` -## Configuration Caddy +### Redis -If you replaced nginx in the previous step, please refer to the "Other reverse proxy server" chapter in the document for configuration. - -1. Run this to modify caddy configuration. - -```sh -sudo nano /etc/caddy/Caddyfile -``` - -2. Add this below, should change `example.tld` to your domain. - -```conf -example.tld { - reverse_proxy http://127.0.0.1:3000 -} -``` - -3. Running `sudo systemctl restart caddy` to apply. - -## Install optional ffmpeg dependencies +Instructions can be found at [this page](https://redis.io/docs/install/install-redis/). ```sh +curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list sudo apt update -sudo apt install -y -V ffmpeg +sudo apt install redis + +sudo systemctl enable --now redis-server + +# check version +redis-cli --version ``` -## Download and configuration Firefish - -1. Download Firefish and Copy example configuration file. +### FFmpeg ```sh -# cd /path/to/your/firefish -git clone https://firefish.dev/firefish/firefish.git -cd firefish/ -git checkout main -cp .config/example.yml .config/default.yml +sudo apt install ffmpeg ``` -> **Note** -> By default, you're on the develop branch. Run `git checkout main` to switch to the Main branch. +## 2. Set up a database -2. Open your `default.yml` files and make changes like `URL`, `db` and `reservedUsernames`. +1. Create a database user + ```sh + sudo -u postgres createuser --no-createdb --no-createrole --no-superuser --encrypted --pwprompt firefish + ``` + If you forgot the password you typed, you can reset it by executing `sudo -u postgres psql -c "ALTER USER firefish PASSWORD 'password';"`. +2. Create a database + ```sh + sudo -u postgres createdb --encoding='UTF8' --owner=firefish firefish_db + ``` +3. Enable PGronnga extension + ```sh + sudo -u postgres psql --command='CREATE EXTENSION pgroonga;' --dbname=firefish_db + ``` + +## 3. Configure Firefish + +1. Create an user for Firefish and switch user + ```sh + sudo useradd --create-home --user-group --shell /bin/bash firefish + sudo su --login firefish + + # check the current working directory + # the result should be /home/firefish + pwd + ``` +1. Install Rust toolchain + Instructions can be found at [this page](https://www.rust-lang.org/tools/install). + + ```sh + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + . "${HOME}/.cargo/env" + + # check version + cargo --version + ``` +3. Clone the Firefish repository + ```sh + git clone --branch=main https://firefish.dev/firefish/firefish.git + ``` +1. Copy and edit the config file + ```sh + cd firefish + cp .config/example.yml .config/default.yml + nano .config/default.yml + ``` + + ```yaml + url: https://your-server-domain.example.com # change here + port: 3000 + + db: + host: localhost + port: 5432 + db: firefish_db + user: firefish + pass: your-database-password # and here + ``` + +## 4. Build Firefish + +1. Build + ```sh + pnpm install --frozen-lockfile + NODE_ENV=production pnpm run build + ``` +1. Execute database migrations + ```sh + pnpm run migrate + ``` +1. Logout from `firefish` user + ```sh + exit + ``` + +## 5. Preparation for publishing a server + +### 1. Set up a firewall + +To expose your server securely, you may want to set up a firewall. We use [ufw](https://launchpad.net/ufw) in this instruction. ```sh -nano .config/default.yml +sudo apt install ufw +# if you use SSH +# SSH_PORT=22 +# sudo ufw limit "${SSH_PORT}/tcp" +sudo ufw default deny +sudo ufw allow 80 +sudo ufw allow 443 +sudo ufw --force enable + +# check status +sudo ufw status ``` -## Install Firefish +### 2. Set up a reverse proxy -**run these steps to update Firefish in the future!** +In this instruction, we use [Caddy](https://caddyserver.com/) to make the Firefish server accesible from internet. However, you can also use [Nginx](https://nginx.org/en/) if you want ([example Nginx config file](../firefish.nginx.conf)). -1. Let corepack enable. +1. Install Caddy + ```sh + sudo apt install debian-keyring debian-archive-keyring apt-transport-https + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list + sudo apt update + sudo apt install caddy + + # check version + caddy version + ``` +1. Replace the config file + ```sh + sudo mv /etc/caddy/Caddyfile /etc/caddy/Caddyfile.bak + sudo nano /etc/caddy/Caddyfile + ``` + + ```Caddyfile + your-server-domain.example.com { + reverse_proxy http://127.0.0.1:3000 + + log { + output file /var/log/caddy/firefish.log + } + } + ``` +1. Restart Caddy + ```sh + sudo systemctl restart caddy + ``` + +## 6. Publish your Firefish server + +1. Create a service file + ```sh + sudo nano /etc/systemd/system/firefish.service + ``` + + ```service + [Unit] + Description=Firefish daemon + Requires=redis.service caddy.service postgresql.service + After=redis.service caddy.service postgresql.service network-online.target + + [Service] + Type=simple + User=firefish + Group=firefish + UMask=0027 + ExecStart=/usr/bin/pnpm run start + WorkingDirectory=/home/firefish/firefish + Environment="NODE_ENV=production" + Environment="npm_config_cache=/tmp" + # uncomment the following line if you use jemalloc (note that the path varies on different environments) + # Environment="LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" + StandardOutput=journal + StandardError=journal + SyslogIdentifier=firefish + TimeoutSec=60 + Restart=always + + CapabilityBoundingSet= + DevicePolicy=closed + NoNewPrivileges=true + LockPersonality=true + PrivateDevices=true + PrivateIPC=true + PrivateMounts=true + PrivateUsers=true + ProtectClock=true + ProtectControlGroups=true + ProtectHostname=true + ProtectKernelTunables=true + ProtectKernelModules=true + ProtectKernelLogs=true + ProtectProc=invisible + RestrictNamespaces=true + RestrictRealtime=true + RestrictSUIDSGID=true + SecureBits=noroot-locked + SystemCallArchitectures=native + SystemCallFilter=~@chown @clock @cpu-emulation @debug @ipc @keyring @memlock @module @mount @obsolete @privileged @raw-io @reboot @resources @setuid @swap + SystemCallFilter=capset pipe pipe2 setpriority + + [Install] + WantedBy=multi-user.target + ``` +1. Start Firefish + ```sh + sudo systemctl enable --now firefish + ``` + +## Upgrading + +Please refer to the [upgrade instruction](./upgrade.md). Be sure to switch to `firefish` user and go to the Firefish directory before performing `pnpm install --frozen-lockfile`: ```sh -# Switching to root -sudo -i -# cd /path/to/your/firefish -cd /home/user/firefish -npm i -g pm2 -corepack enable -exit -``` - -2. Install dependency. - -```sh -corepack prepare pnpm@latest --activate -pnpm install --frozen-lockfile --prod false -pm2 install pm2-logrotate -``` - -> **Note** -> [`pm2-logrotate`](https://github.com/keymetrics/pm2-logrotate/blob/master/README.md) ensures that log files don't infinitely gather size, as Firefish produces a lot of logs. - -3. Build and migrate - -```sh -pnpm install --prod false -NODE_ENV=production pnpm run build && pnpm run migrate -``` - -4. Start Firefish - -```sh -pm2 start "NODE_ENV=production pnpm run start" --name Firefish -pm2 logs Firefish -``` - -5. Wait until the following message shows up. - -```log -1|Firefish | DONE * [core boot] All workers started -1|Firefish | DONE * [core boot] Now listening on port 3000 on https://your_firefish_url.example.com (default value: https://localhost:3000) -``` - -6. A fresh Firefish environment is created on the URL you have set! - -7. By the way, Please use content at below to generate vapid keys to enable Push-Notifications. - -```sh -# Switching to root -sudo -i -npm install -g web-push -web-push generate-vapid-keys -exit +sudo su --login firefish +cd ~/firefish ``` ## Customize @@ -263,22 +299,13 @@ exit - To update custom assets without rebuilding, just run `pnpm run gulp`. - To block ChatGPT, CommonCrawl, or other crawlers from indexing your instance, uncomment the respective rules in `./custom/robots.txt`. -## Other reverse proxy server - -### Nginx - -- Run `sudo cp ./firefish.nginx.conf /etc/nginx/sites-available/ && cd /etc/nginx/sites-available/` . -- Edit `firefish.nginx.conf` to reflect your server properly. -- Run `sudo ln -s ./firefish.nginx.conf ../sites-enabled/firefish.nginx.conf` . -- Run `sudo nginx -t` to validate that the config is valid, then restart the NGINX service. - ## Tips & Tricks - When editing the config file, please don't fill out the settings at the bottom. They're designed *only* for managed hosting, not self hosting. Those settings are much better off being set in Firefish's control panel. - Port 3000 (used in the default config) might be already used on your server for something else. To find an open port for Firefish, run `for p in {3000..4000}; do ss -tlnH | tr -s ' ' | cut -d" " -sf4 | grep -q "${p}$" || echo "${p}"; done | head -n 1`. Replace 3000 with the minimum port and 4000 with the maximum port if you need it. -- I'd recommend you use a S3 Bucket/CDN for Object Storage, especially if you use Docker. +- We'd recommend you use a S3 Bucket/CDN for Object Storage, especially if you use containers. - When using object storage, setting a proper `Access-Control-Allow-Origin` response header is highly recommended. -- I'd ***strongly*** recommend against using CloudFlare, but if you do, make sure to turn code minification off. +- We'd recommend against using CloudFlare, but if you do, make sure to turn code minification off. - For push notifications, run `npx web-push generate-vapid-keys`, then put the public and private keys into Control Panel > General > ServiceWorker. - For translations, make a [DeepL](https://deepl.com) account and generate an API key, then put it into Control Panel > General > DeepL Translation. - To add another admin account: diff --git a/docs/notice-for-admins.md b/docs/notice-for-admins.md index c52cf61620..f763bf02ae 100644 --- a/docs/notice-for-admins.md +++ b/docs/notice-for-admins.md @@ -1,13 +1,17 @@ -# Unreleased +# Notice for server administrators + +You can skip intermediate versions when upgrading from an old version, but please read the notices and follow the instructions for each intermediate version before [upgrading](./upgrade.md). + +## Unreleased The full-text search engine used in Firefish has been changed to [PGroonga](https://pgroonga.github.io/). This is no longer an optional feature, so please enable PGroonga on your system. If you are using Sonic, Meilisearch, or Elasticsearch, you can also uninstall it from your system and remove the settings from `.config/default.yml`. -## For systemd/pm2 users +### For systemd/pm2 users - Required Node.js version has been bumped from v18.16.0 to v18.17.0. - You need to install PGroonga on your system. Please follow the instructions below. -### 1. Install PGroonga +#### 1. Install PGroonga Please execute `psql --version` to check your PostgreSQL major version. This will print a message like this: @@ -19,7 +23,7 @@ In this case, your PostgreSQL major version is `16`. There are official installation instructions for many operating systems on , so please follow the instructions on this page. However, since many users are using Ubuntu, and there are no instructions for Arch Linux, we explicitly list the instructions for Ubuntu and Arch Linux here. Please keep in mind that this is not official information and the procedures may change. -#### Ubuntu +##### Ubuntu 1. Add apt repository ```sh @@ -39,7 +43,7 @@ There are official installation instructions for many operating systems on -# v20240214 +## v20240214 -## For systemd/pm2 users +### For systemd/pm2 users - Required Rust version has been bumped from v1.70 to v1.74. ```sh @@ -156,9 +160,9 @@ A new setting item has been added to control the log levels, so please consider rustup update # update version ``` -# v20240213 +## v20240213 -## For systemd/pm2 users +### For systemd/pm2 users - `packages/backend/native-utils` can be removed. - This directory was removed in the repository, but it's not completely removed from your system by `git pull --ff`, because some folders like `packages/backend/native-utils/built` are not tracked by git. @@ -167,16 +171,16 @@ A new setting item has been added to control the log levels, so please consider rm --recursive --force packages/backend/native-utils ``` -# v20240206 +## v20240206 -## For all users +### For all users - The git repository has been moved, so please update the `git remote` url. ```sh git remote set-url origin https://firefish.dev/firefish/firefish.git ``` -## For systemd/pm2 users +### For systemd/pm2 users - Required Rust version has been bumped from v1.68 to v1.70. - `libvips` is no longer required (unless your server os is *BSD), so you may uninstall it from your system. Make sure to execute the following commands after that: @@ -185,6 +189,6 @@ A new setting item has been added to control the log levels, so please consider pnpm install ``` -## For Docker/Podman users +### For Docker/Podman users - The image tag has been changed to `registry.firefish.dev/firefish/firefish:latest`, so please update `docker-compose.yml`. diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 0000000000..2d0c6c5a10 --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,41 @@ +# Upgrade instruction + +## For systemd/pm2 users + +1. Check [`docs/notice-for-admins.md`](./notice-for-admins.md) +1. Stop the server + ```sh + sudo systemctl stop your-firefish-service.service + # or pm2 stop firefish + ``` +1. Pull the latest source code + ```sh + git checkout -- packages/backend/assets + git pull --ff origin main + ``` +1. Build Firefish and apply changes to the database + ```sh + corepack prepare pnpm@latest --activate + pnpm install --frozen-lockfile + NODE_ENV='production' NODE_OPTIONS='--max_old_space_size=3072' pnpm run rebuild + pnpm run migrate + ``` +1. Start the server + ```sh + sudo systemctl start your-firefish-service.service + # or pm2 start firefish + ``` + +## For Docker/Podman users + +1. Check [`docs/notice-for-admins.md`](./notice-for-admins.md) +1. Pull the latest container image + ```sh + docker pull registry.firefish.dev/firefish/firefish:latest + # or podman pull registry.firefish.dev/firefish/firefish:latest + ``` +1. Start the container + ```sh + docker compose up --detach + # or podman-compose up --detach + ``` diff --git a/scripts/db-down.mjs b/scripts/db-down.mjs index bb972b262b..21de3f5a54 100644 --- a/scripts/db-down.mjs +++ b/scripts/db-down.mjs @@ -6,7 +6,7 @@ import { execa } from "execa"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); execa("podman-compose", ["down"], { - cwd: join(__dirname, "/../dev/docker-dbonly"), + cwd: join(__dirname, "/../dev/db-container"), stdio: "inherit", }); })(); diff --git a/scripts/db-up.mjs b/scripts/db-up.mjs index 08842c2eb9..e7af479b3f 100644 --- a/scripts/db-up.mjs +++ b/scripts/db-up.mjs @@ -6,7 +6,7 @@ import { execa } from "execa"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); execa("podman-compose", ["up", "--detach"], { - cwd: join(__dirname, "/../dev/docker-dbonly"), + cwd: join(__dirname, "/../dev/db-container"), stdio: "inherit", }); })();