Introduce processor

This commit is contained in:
Akihiko Odaki 2018-03-29 01:20:40 +09:00
parent 68ce6d5748
commit 90f8fe7e53
582 changed files with 246 additions and 188 deletions

View file

@ -22,7 +22,7 @@ import * as replace from 'gulp-replace';
import * as htmlmin from 'gulp-htmlmin';
const uglifyes = require('uglify-es');
import { fa } from './src/common/build/fa';
import { fa } from './src/build/fa';
import version from './src/version';
import config from './src/conf';
@ -39,7 +39,7 @@ if (isDebug) {
const constants = require('./src/const.json');
require('./src/web/docs/gulpfile.ts');
require('./src/server/web/docs/gulpfile.ts');
gulp.task('build', [
'build:js',
@ -52,7 +52,7 @@ gulp.task('build', [
gulp.task('rebuild', ['clean', 'build']);
gulp.task('build:js', () =>
gulp.src(['./src/**/*.js', '!./src/web/**/*.js'])
gulp.src(['./src/**/*.js', '!./src/server/web/**/*.js'])
.pipe(gulp.dest('./built/'))
);
@ -71,7 +71,7 @@ gulp.task('build:copy', () =>
gulp.src([
'./build/Release/crypto_key.node',
'./src/**/assets/**/*',
'!./src/web/app/**/assets/**/*'
'!./src/server/web/app/**/assets/**/*'
]).pipe(gulp.dest('./built/'))
);
@ -121,7 +121,7 @@ gulp.task('build:client', [
]);
gulp.task('build:client:script', () =>
gulp.src(['./src/web/app/boot.js', './src/web/app/safe.js'])
gulp.src(['./src/server/web/app/boot.js', './src/server/web/app/safe.js'])
.pipe(replace('VERSION', JSON.stringify(version)))
.pipe(replace('API', JSON.stringify(config.api_url)))
.pipe(replace('ENV', JSON.stringify(env)))
@ -129,15 +129,15 @@ gulp.task('build:client:script', () =>
.pipe(isProduction ? uglify({
toplevel: true
} as any) : gutil.noop())
.pipe(gulp.dest('./built/web/assets/')) as any
.pipe(gulp.dest('./built/server/web/assets/')) as any
);
gulp.task('build:client:styles', () =>
gulp.src('./src/web/app/init.css')
gulp.src('./src/server/web/app/init.css')
.pipe(isProduction
? (cssnano as any)()
: gutil.noop())
.pipe(gulp.dest('./built/web/assets/'))
.pipe(gulp.dest('./built/server/web/assets/'))
);
gulp.task('copy:client', [
@ -145,14 +145,14 @@ gulp.task('copy:client', [
], () =>
gulp.src([
'./assets/**/*',
'./src/web/assets/**/*',
'./src/web/app/*/assets/**/*'
'./src/server/web/assets/**/*',
'./src/server/web/app/*/assets/**/*'
])
.pipe(isProduction ? (imagemin as any)() : gutil.noop())
.pipe(rename(path => {
path.dirname = path.dirname.replace('assets', '.');
}))
.pipe(gulp.dest('./built/web/assets/'))
.pipe(gulp.dest('./built/server/web/assets/'))
);
gulp.task('build:client:pug', [
@ -160,13 +160,13 @@ gulp.task('build:client:pug', [
'build:client:script',
'build:client:styles'
], () =>
gulp.src('./src/web/app/base.pug')
gulp.src('./src/server/web/app/base.pug')
.pipe(pug({
locals: {
themeColor: constants.themeColor,
facss: fa.dom.css(),
//hljscss: fs.readFileSync('./node_modules/highlight.js/styles/default.css', 'utf8')
hljscss: fs.readFileSync('./src/web/assets/code-highlight.css', 'utf8')
hljscss: fs.readFileSync('./src/server/web/assets/code-highlight.css', 'utf8')
}
}))
.pipe(htmlmin({
@ -201,5 +201,5 @@ gulp.task('build:client:pug', [
// CSSも圧縮する
minifyCSS: true
}))
.pipe(gulp.dest('./built/web/app/'))
.pipe(gulp.dest('./built/server/web/app/'))
);

View file

@ -135,6 +135,7 @@
"is-url": "1.2.3",
"js-yaml": "3.11.0",
"jsdom": "^11.6.2",
"kue": "^0.11.6",
"license-checker": "18.0.0",
"loader-utils": "1.1.0",
"mecab-async": "0.1.2",
@ -149,6 +150,7 @@
"nan": "^2.10.0",
"node-sass": "4.7.2",
"node-sass-json-importer": "3.1.5",
"nopt": "^4.0.1",
"nprogress": "0.2.0",
"object-assign-deep": "0.3.1",
"on-build-webpack": "0.1.0",

View file

@ -2,7 +2,7 @@
* Replace i18n texts
*/
import locale from '../../../locales';
import locale from '../../locales';
export default class Replacer {
private lang: string;

View file

@ -1,6 +1,6 @@
import * as fs from 'fs';
const license = fs.readFileSync(__dirname + '/../../../LICENSE', 'utf-8');
const license = fs.readFileSync(__dirname + '/../../LICENSE', 'utf-8');
const licenseHtml = license
.replace(/\r\n/g, '\n')

View file

@ -24,6 +24,8 @@ import stats from './utils/stats';
import { Config, path as configPath } from './config';
import loadConfig from './config';
import parseOpt from './parse-opt';
const clusterLog = debug('misskey:cluster');
const ev = new Xev();
@ -36,20 +38,22 @@ main();
* Init process
*/
function main() {
const opt = parseOpt(process.argv, 2);
if (cluster.isMaster) {
masterMain();
masterMain(opt);
ev.mount();
stats();
} else {
workerMain();
workerMain(opt);
}
}
/**
* Init master process
*/
async function masterMain() {
async function masterMain(opt) {
let config: Config;
try {
@ -69,19 +73,35 @@ async function masterMain() {
}
spawnWorkers(() => {
Logger.info(chalk.bold.green(
`Now listening on port ${chalk.underline(config.port.toString())}`));
if (!opt['only-processor']) {
Logger.info(chalk.bold.green(
`Now listening on port ${chalk.underline(config.port.toString())}`));
Logger.info(chalk.bold.green(config.url));
Logger.info(chalk.bold.green(config.url));
}
if (!opt['only-server']) {
Logger.info(chalk.bold.green('Now processing jobs'));
}
});
}
/**
* Init worker process
*/
function workerMain() {
// start server
require('./server');
async function workerMain(opt) {
if (!opt['only-processor']) {
// start server
await require('./server').default();
}
if (!opt['only-server']) {
// start processor
require('./processor').default();
}
// Send a 'ready' message to parent process
process.send('ready');
}
/**

17
src/parse-opt.ts Normal file
View file

@ -0,0 +1,17 @@
const nopt = require('nopt');
export default (vector, index) => {
const parsed = nopt({
'only-processor': Boolean,
'only-server': Boolean
}, {
p: ['--only-processor'],
s: ['--only-server']
}, vector, index);
if (parsed['only-processor'] && parsed['only-server']) {
throw 'only-processor option and only-server option cannot be set at the same time';
}
return parsed;
};

4
src/processor/index.ts Normal file
View file

@ -0,0 +1,4 @@
import queue from '../queue';
import reportGitHubFailure from './report-github-failure';
export default () => queue.process('gitHubFailureReport', reportGitHubFailure);

View file

@ -0,0 +1,29 @@
import * as request from 'request';
import User from '../server/api/models/user';
const createPost = require('../server/api/endpoints/posts/create');
export default ({ data }, done) => {
const asyncBot = User.findOne({ _id: data.userId });
// Fetch parent status
request({
url: `${data.parentUrl}/statuses`,
headers: {
'User-Agent': 'misskey'
}
}, async (err, res, body) => {
if (err) {
console.error(err);
return;
}
const parentStatuses = JSON.parse(body);
const parentState = parentStatuses[0].state;
const stillFailed = parentState == 'failure' || parentState == 'error';
const text = stillFailed ?
`**⚠BUILD STILL FAILED⚠**: ?[${data.message}](${data.htmlUrl})` :
`**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`;
createPost({ text }, await asyncBot);
done();
});
};

10
src/queue.ts Normal file
View file

@ -0,0 +1,10 @@
import { createQueue } from 'kue';
import config from './conf';
export default createQueue({
redis: {
port: config.redis.port,
host: config.redis.host,
auth: config.redis.pass
}
});

View file

@ -3,9 +3,9 @@ import * as express from 'express';
import * as request from 'request';
import * as crypto from 'crypto';
import User from '../../models/user';
import config from '../../../conf';
import config from '../../../../conf';
import BotCore from '../core';
import _redis from '../../../db/redis';
import _redis from '../../../../db/redis';
import prominence = require('prominence');
import getAcct from '../../../common/user/get-acct';
import parseAcct from '../../../common/user/parse-acct';

View file

@ -15,7 +15,7 @@ import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file';
import event, { publishDriveStream } from '../../event';
import getAcct from '../../../common/user/get-acct';
import config from '../../../conf';
import config from '../../../../conf';
const gm = _gm.subClass({
imageMagick: true

View file

@ -1,7 +1,7 @@
const push = require('web-push');
import * as mongo from 'mongodb';
import Subscription from '../models/sw-subscription';
import config from '../../conf';
import config from '../../../conf';
if (config.sw) {
// アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録

View file

@ -1,4 +1,4 @@
import config from '../../conf';
import config from '../../../conf';
export default function(res, user, redirect: boolean) {
const expires = 1000 * 60 * 60 * 24 * 365; // One Year

View file

@ -5,7 +5,7 @@ import * as uuid from 'uuid';
import $ from 'cafy';
import App from '../../../models/app';
import AuthSess from '../../../models/auth-session';
import config from '../../../../conf';
import config from '../../../../../conf';
/**
* @swagger

View file

@ -6,7 +6,7 @@ import * as bcrypt from 'bcryptjs';
import * as speakeasy from 'speakeasy';
import * as QRCode from 'qrcode';
import User from '../../../models/user';
import config from '../../../../conf';
import config from '../../../../../conf';
module.exports = async (params, user) => new Promise(async (res, rej) => {
// Get 'password' parameter

View file

@ -4,7 +4,7 @@
import $ from 'cafy';
import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../models/user';
import event from '../../event';
import config from '../../../conf';
import config from '../../../../conf';
/**
* Update myself

View file

@ -11,7 +11,7 @@ import DriveFile from '../../../models/drive-file';
import { pack } from '../../../models/messaging-message';
import publishUserStream from '../../../event';
import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event';
import config from '../../../../conf';
import config from '../../../../../conf';
/**
* Create a message

View file

@ -2,8 +2,8 @@
* Module dependencies
*/
import * as os from 'os';
import version from '../../version';
import config from '../../conf';
import version from '../../../version';
import config from '../../../conf';
import Meta from '../models/meta';
/**

Some files were not shown because too many files have changed in this diff Show more