Start work on OCR

2022-08-08 15:40:06 -07:00 · 2022-08-08 15:40:06 -07:00 · 98721d0dcd
parent 795ed6ff85
commit 98721d0dcd
8 changed files with 181 additions and 37 deletions
--- a/CALCKEY.md
+++ b/CALCKEY.md
@ -20,6 +20,7 @@

 ## Work in progress

+- OCR image/video captioning
 - Recommended instances timeline
 - Admin custom CSS
 - Improve accesibility score
--- a/locales/en-US.yml
+++ b/locales/en-US.yml
@ -911,6 +911,7 @@ customSplashIconsDescription: "URLs for custom splash screen icons separated by
 showUpdates: "Show a popup when Calckey updates"
 recommendedInstances: "Recommended instances"
 recommendedInstancesDescription: "Recommended instances seperated by line breaks to appear in the recommended timeline. Do NOT add `https://`, ONLY the domain."
+caption: "Auto Caption"

 _sensitiveMediaDetection:
  description: "Reduces the effort of server moderation through automatically recognizing NSFW media via Machine Learning. This will slightly increase the load on the server."
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@ -911,6 +911,7 @@ customSplashIconsDescription: "ユーザがページをロード/リロードす
 showUpdates: "Calckeyの更新時にポップアップを表示する"
 recommendedInstances: "推奨インスタンス"
 recommendedInstancesDescription: "推奨タイムラインに表示するために改行で区切られた推奨インスタンス。`https：//`を追加しないでください。ドメインのみを追加してください。"
+caption: "自動キャプション"

 _sensitiveMediaDetection:
  description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。"
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
 	"name": "misskey",
-	"version": "12.118.1-calc.rc.3",
+	"version": "12.118.1-calc.rc.4",
 	"codename": "aqua",
 	"repository": {
 		"type": "git",
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -47,6 +47,7 @@
 		"strict-event-emitter-types": "2.0.0",
 		"stringz": "2.1.0",
 		"syuilo-password-strength": "0.0.1",
+		"tesseract.js": "^2.1.5",
 		"textarea-caret": "3.1.0",
 		"three": "0.143.0",
 		"throttle-debounce": "5.0.0",
--- a/packages/client/src/components/gallery-post-preview.vue
+++ b/packages/client/src/components/gallery-post-preview.vue
@ -31,7 +31,7 @@ export default defineComponent({
 		},
 	},
 	methods: {
-		userName
+		userName,
 	}
 });
 </script>
--- a/packages/client/src/components/media-caption.vue
+++ b/packages/client/src/components/media-caption.vue
@ -1,40 +1,46 @@
 <template>
-	<MkModal ref="modal" @click="done(true)" @closed="$emit('closed')">
-		<div class="container">
-			<div class="fullwidth top-caption">
-				<div class="mk-dialog">
-					<header>
-						<Mfm v-if="title" class="title" :text="title"/>
-						<span class="text-count" :class="{ over: remainingLength < 0 }">{{ remainingLength }}</span>
-					</header>
-					<textarea v-model="inputValue" autofocus :placeholder="input.placeholder" @keydown="onInputKeydown"></textarea>
-					<div v-if="(showOkButton || showCancelButton)" class="buttons">
-						<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ $ts.ok }}</MkButton>
-						<MkButton inline @click="cancel" >{{ $ts.cancel }}</MkButton>
-					</div>
+<MkModal ref="modal" @click="done(true)" @closed="$emit('closed')">
+	<div class="container">
+		<div class="fullwidth top-caption">
+			<div class="mk-dialog">
+				<header>
+					<Mfm v-if="title" class="title" :text="title"/>
+					<span class="text-count" :class="{ over: remainingLength < 0 }">{{ remainingLength }}</span>
+				</header>
+				<textarea v-model="inputValue" autofocus :placeholder="input.placeholder" @keydown="onInputKeydown"></textarea>
+				<div v-if="(showOkButton || showCaptionButton || showCancelButton)" class="buttons">
+					<MkButton inline primary :disabled="remainingLength < 0" @click="ok">{{ $ts.ok }}</MkButton>
+					<MkButton inline @click="caption" >{{ $ts.caption }}</MkButton>
+					<MkButton inline @click="cancel" >{{ $ts.cancel }}</MkButton>
 				</div>
 			</div>
-			<div class="hdrwpsaf fullwidth">
-				<header>{{ image.name }}</header>
-				<img :src="image.url" :alt="image.comment" :title="image.comment" @click="$refs.modal.close()"/>
-				<footer>
-					<span>{{ image.type }}</span>
-					<span>{{ bytes(image.size) }}</span>
-					<span v-if="image.properties && image.properties.width">{{ number(image.properties.width) }}px × {{ number(image.properties.height) }}px</span>
-				</footer>
-			</div>
 		</div>
-	</MkModal>
+		<div class="hdrwpsaf fullwidth">
+			<header>{{ image.name }}</header>
+			<img id="imgtocaption" :src="image.url" :alt="image.comment" :title="image.comment" @click="$refs.modal.close()"/>
+			<footer>
+				<span>{{ image.type }}</span>
+				<span>{{ bytes(image.size) }}</span>
+				<span v-if="image.properties && image.properties.width">{{ number(image.properties.width) }}px × {{ number(image.properties.height) }}px</span>
+			</footer>
+		</div>
+	</div>
+</MkModal>
 </template>

 <script lang="ts">
 import { defineComponent } from 'vue';
 import { length } from 'stringz';
+import { createWorker, PSM, OEM } from 'tesseract.js';
 import MkModal from '@/components/ui/modal.vue';
 import MkButton from '@/components/ui/button.vue';
 import bytes from '@/filters/bytes';
 import number from '@/filters/number';

+const worker = createWorker({
+	logger: m => console.log(m),
+});
+
 export default defineComponent({
 	components: {
 		MkModal,
@ -48,22 +54,26 @@ export default defineComponent({
 		},
 		title: {
 			type: String,
-			required: false
+			required: false,
 		},
 		input: {
-			required: true
+			required: true,
 		},
 		showOkButton: {
 			type: Boolean,
-			default: true
+			default: true,
+		},
+		showCaptionButton: {
+			type: Boolean,
+			default: true,
 		},
 		showCancelButton: {
 			type: Boolean,
-			default: true
+			default: true,
 		},
 		cancelableByBgClick: {
 			type: Boolean,
-			default: true
+			default: true,
 		},
 	},

@ -93,6 +103,8 @@ export default defineComponent({
 	methods: {
 		bytes,
 		number,
+		PSM,
+		OEM,

 		done(canceled, result?) {
 			this.$emit('done', { canceled, result });
@ -130,8 +142,20 @@ export default defineComponent({
 					this.ok();
 				}
 			}
-		}
-	}
+		},
+
+		caption: async () => {
+			const img = document.getElementById('imgtocaption');
+			await worker.load();
+			await worker.loadLanguage('eng');
+			await worker.initialize('eng', OEM.LSTM_ONLY);
+			await worker.setParameters({
+				tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
+			});
+			const { data: { text } } = await worker.recognize(img);
+			console.log(text);
+		},
+	},
 });
 </script>

--- a/yarn.lock
+++ b/yarn.lock
@ -3210,6 +3210,13 @@ __metadata:
  languageName: node
  linkType: hard

+"blueimp-load-image@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "blueimp-load-image@npm:3.0.0"
+  checksum: 19a1e9a76ea87e15c1624c3ed30536f094252f0a65a193239f54ec0c25fe480b5140c962ac41a66f35a2d19b58d9da26803287b5a37b78d82d4b23328b09bf4a
+  languageName: node
+  linkType: hard
+
 "blurhash@npm:1.1.5":
  version: 1.1.5
  resolution: "blurhash@npm:1.1.5"
@ -3217,6 +3224,13 @@ __metadata:
  languageName: node
  linkType: hard

+"bmp-js@npm:^0.1.0":
+  version: 0.1.0
+  resolution: "bmp-js@npm:0.1.0"
+  checksum: 2f6cf7eeabae2aa50eb768122f59e9752caa97248028cb8b5cf0d9db7cf8fb3a60262aeb2c6889dd21357ab061b2fb318f21f20d2b24963ba36ead8e264c6654
+  languageName: node
+  linkType: hard
+
 "bn.js@npm:^4.0.0":
  version: 4.12.0
  resolution: "bn.js@npm:4.12.0"
@ -3950,6 +3964,7 @@ __metadata:
    strict-event-emitter-types: 2.0.0
    stringz: 2.1.0
    syuilo-password-strength: 0.0.1
+    tesseract.js: ^2.1.5
    textarea-caret: 3.1.0
    three: 0.143.0
    throttle-debounce: 5.0.0
@ -4229,6 +4244,13 @@ __metadata:
  languageName: node
  linkType: hard

+"colors@npm:^1.4.0":
+  version: 1.4.0
+  resolution: "colors@npm:1.4.0"
+  checksum: 98aa2c2418ad87dedf25d781be69dc5fc5908e279d9d30c34d8b702e586a0474605b3a189511482b9d5ed0d20c867515d22749537f7bc546256c6014f3ebdcec
+  languageName: node
+  linkType: hard
+
 "colors@npm:~1.1.2":
  version: 1.1.2
  resolution: "colors@npm:1.1.2"
@ -6385,6 +6407,13 @@ __metadata:
  languageName: node
  linkType: hard

+"file-type@npm:^12.4.1":
+  version: 12.4.2
+  resolution: "file-type@npm:12.4.2"
+  checksum: 67c8d7f8f032fd8cf4d14016d96567d20eeb7bf3524915f2c5d79337ca4e5338032d373a5fe827610eaf4ab7eb80629ff868331a66f63d1f9e9cc4c433e3f047
+  languageName: node
+  linkType: hard
+
 "filelist@npm:^1.0.1":
  version: 1.0.4
  resolution: "filelist@npm:1.0.4"
@ -7036,7 +7065,7 @@ __metadata:
  languageName: node
  linkType: hard

-"glob@npm:^7.1.1, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.2.0":
+"glob@npm:^7.1.1, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.2.0":
  version: 7.2.3
  resolution: "glob@npm:7.2.3"
  dependencies:
@ -7708,6 +7737,13 @@ __metadata:
  languageName: node
  linkType: hard

+"idb-keyval@npm:^3.2.0":
+  version: 3.2.0
+  resolution: "idb-keyval@npm:3.2.0"
+  checksum: 3270010af049fa7f07b0381818daa7f16d3704aaba9d2585e531908f09130f6d0772464b2b48c23676dd1e51e5245585da5ce56f2c74dfd4f9961d1e0d0f5909
+  languageName: node
+  linkType: hard
+
 "ieee754@npm:1.1.13":
  version: 1.1.13
  resolution: "ieee754@npm:1.1.13"
@ -8098,6 +8134,13 @@ __metadata:
  languageName: node
  linkType: hard

+"is-electron@npm:^2.2.0":
+  version: 2.2.1
+  resolution: "is-electron@npm:2.2.1"
+  checksum: 06e569aa933a737d418489bb9ca081af62eceb714d4c3d553ad2497610e35494be6dddd010c4e29890c7dd9d0481c2e3e1e9097af9d19df1c52dd5be747d80a0
+  languageName: node
+  linkType: hard
+
 "is-expression@npm:^4.0.0":
  version: 4.0.0
  resolution: "is-expression@npm:4.0.0"
@ -8411,6 +8454,13 @@ __metadata:
  languageName: node
  linkType: hard

+"is-url@npm:^1.2.4":
+  version: 1.2.4
+  resolution: "is-url@npm:1.2.4"
+  checksum: 100e74b3b1feab87a43ef7653736e88d997eb7bd32e71fd3ebc413e58c1cbe56269699c776aaea84244b0567f2a7d68dfaa512a062293ed2f9fdecb394148432
+  languageName: node
+  linkType: hard
+
 "is-utf8@npm:^0.2.0, is-utf8@npm:^0.2.1":
  version: 0.2.1
  resolution: "is-utf8@npm:0.2.1"
@ -8536,6 +8586,21 @@ __metadata:
  languageName: node
  linkType: hard

+"jpeg-autorotate@npm:^7.1.1":
+  version: 7.1.1
+  resolution: "jpeg-autorotate@npm:7.1.1"
+  dependencies:
+    colors: ^1.4.0
+    glob: ^7.1.6
+    jpeg-js: ^0.4.2
+    piexifjs: ^1.0.6
+    yargs-parser: ^20.2.1
+  bin:
+    jpeg-autorotate: src/cli.js
+  checksum: fdd6898ed287e074e26667e503731846ddb3e3042d5aee70f3f6ae86051b8d493bfa0906f9df19a4e5fb0df6253ff059bf8967cda5fb3e6f1c8e8e2d7a58a3ee
+  languageName: node
+  linkType: hard
+
 "jpeg-js@npm:^0.3.2":
  version: 0.3.7
  resolution: "jpeg-js@npm:0.3.7"
@ -8543,7 +8608,7 @@ __metadata:
  languageName: node
  linkType: hard

-"jpeg-js@npm:^0.4.1":
+"jpeg-js@npm:^0.4.1, jpeg-js@npm:^0.4.2":
  version: 0.4.4
  resolution: "jpeg-js@npm:0.4.4"
  checksum: bd7cb61aa8df40a9ee2c2106839c3df6054891e56cfc22c0ac581402e06c6295f962a4754b0b2ac50a401789131b1c6dc9df8d24400f1352168be1894833c590
@ -10316,7 +10381,7 @@ __metadata:
  languageName: node
  linkType: hard

-"node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:~2.6.1":
+"node-fetch@npm:^2.6.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:~2.6.1":
  version: 2.6.7
  resolution: "node-fetch@npm:2.6.7"
  dependencies:
@ -10728,6 +10793,15 @@ __metadata:
  languageName: node
  linkType: hard

+"opencollective-postinstall@npm:^2.0.2":
+  version: 2.0.3
+  resolution: "opencollective-postinstall@npm:2.0.3"
+  bin:
+    opencollective-postinstall: index.js
+  checksum: 0a68c5cef135e46d11e665d5077398285d1ce5311c948e8327b435791c409744d4a6bb9c55bd6507fb5f2ef34b0ad920565adcdaf974cbdae701aead6f32b396
+  languageName: node
+  linkType: hard
+
 "opentype.js@npm:^0.4.3":
  version: 0.4.11
  resolution: "opentype.js@npm:0.4.11"
@ -11262,6 +11336,13 @@ __metadata:
  languageName: node
  linkType: hard

+"piexifjs@npm:^1.0.6":
+  version: 1.0.6
+  resolution: "piexifjs@npm:1.0.6"
+  checksum: 137caf5ca36c0ad98cfd445c0ec40737f7589469e750ac417a2019b940cf6534e9834f3f01dc6a85a6d6c127c3eb5b5dc750812c800e6567392b67da94a755bd
+  languageName: node
+  linkType: hard
+
 "pify@npm:^2.0.0, pify@npm:^2.2.0":
  version: 2.3.0
  resolution: "pify@npm:2.3.0"
@ -12412,7 +12493,7 @@ __metadata:
  languageName: node
  linkType: hard

-"regenerator-runtime@npm:^0.13.4, regenerator-runtime@npm:^0.13.5":
+"regenerator-runtime@npm:^0.13.3, regenerator-runtime@npm:^0.13.4, regenerator-runtime@npm:^0.13.5":
  version: 0.13.9
  resolution: "regenerator-runtime@npm:0.13.9"
  checksum: 65ed455fe5afd799e2897baf691ca21c2772e1a969d19bb0c4695757c2d96249eb74ee3553ea34a91062b2a676beedf630b4c1551cc6299afb937be1426ec55e
@ -13895,6 +13976,34 @@ __metadata:
  languageName: node
  linkType: hard

+"tesseract.js-core@npm:^2.2.0":
+  version: 2.2.0
+  resolution: "tesseract.js-core@npm:2.2.0"
+  checksum: 7587514a6bd51540ba7bbe1fd6349a83c5865261db43bee1625b07c877ac6949b1bf7e0a9a84b6925f8acfb34287f9c28dfd3f3d1bbaa49abc9cbb6cb7927389
+  languageName: node
+  linkType: hard
+
+"tesseract.js@npm:^2.1.5":
+  version: 2.1.5
+  resolution: "tesseract.js@npm:2.1.5"
+  dependencies:
+    blueimp-load-image: ^3.0.0
+    bmp-js: ^0.1.0
+    file-type: ^12.4.1
+    idb-keyval: ^3.2.0
+    is-electron: ^2.2.0
+    is-url: ^1.2.4
+    jpeg-autorotate: ^7.1.1
+    node-fetch: ^2.6.0
+    opencollective-postinstall: ^2.0.2
+    regenerator-runtime: ^0.13.3
+    resolve-url: ^0.2.1
+    tesseract.js-core: ^2.2.0
+    zlibjs: ^0.3.1
+  checksum: a28a2edae162ab5db97a9337dbce1ea96f3c303f6976560784d9f774be6fec15b05f04b8776fe6b9d12471324ee0c94fe25efd2d991fdb859ad66d24e8d66fb0
+  languageName: node
+  linkType: hard
+
 "text-table@npm:^0.2.0":
  version: 0.2.0
  resolution: "text-table@npm:0.2.0"
@ -15433,7 +15542,7 @@ __metadata:
  languageName: node
  linkType: hard

-"yargs-parser@npm:^20.2.2":
+"yargs-parser@npm:^20.2.1, yargs-parser@npm:^20.2.2":
  version: 20.2.9
  resolution: "yargs-parser@npm:20.2.9"
  checksum: 8bb69015f2b0ff9e17b2c8e6bfe224ab463dd00ca211eece72a4cd8a906224d2703fb8a326d36fdd0e68701e201b2a60ed7cf81ce0fd9b3799f9fe7745977ae3
@ -15573,3 +15682,10 @@ __metadata:
  checksum: 4a73da856738b0634700b52f4ab3fe0bf0a532bea6820ad962d0bda0163d2d5525df4859f89a7238e204a378384e12551985049790c1894c3ac191866e85887f
  languageName: node
  linkType: hard
+
+"zlibjs@npm:^0.3.1":
+  version: 0.3.1
+  resolution: "zlibjs@npm:0.3.1"
+  checksum: 91f3d28bb5925ac71565a1fd4d44303b6b876ed483c9c192864393811151b399f29d917c9118a5aca5a541d9dfc7c199460f86b489ea1f77b7217e0edc887c89
+  languageName: node
+  linkType: hard