+'use strict';+import { QueryInterface, DataTypes } from 'sequelize';++module.exports = {

+'use strict';+import {QueryInterface, DataTypes} from 'sequelize';++module.exports = {+  up: async (queryInterface: QueryInterface) => {+    return queryInterface.sequelize.transaction(t => {+      return Promise.all([+        queryInterface.addColumn('pipelines', 'datasetProcess', {+          type: DataTypes.STRING+        }, { transaction: t }),+        queryInterface.addColumn('pipelines', 'datasetProcessParams', {+          type: DataTypes.STRING+        }, { transaction: t })+      ])+    })+  },++  down: (queryInterface: QueryInterface) => {+    return queryInterface.sequelize.transaction(t => {+      return queryInterface.removeColumn('pipelines', 'datasetProcess', { transaction: t }).then(() => {

   },   "scripts": {     "postinstall": "lerna bootstrap",-    "build": "lerna run compile",+    "build": "lerna run compile && rm ./packages/daemon/dist/migrations/*.d.ts",

+'use strict';+import {QueryInterface, DataTypes} from 'sequelize';++module.exports = {+  up: async (queryInterface: QueryInterface) => {+    return queryInterface.sequelize.transaction(t => {+      return Promise.all([+        queryInterface.addColumn('pipelines', 'datasetProcess', {+          type: DataTypes.STRING+        }, { transaction: t }),+        queryInterface.addColumn('pipelines', 'datasetProcessParams', {+          type: DataTypes.STRING+        }, { transaction: t })+      ])+    })+  },++  down: (queryInterface: QueryInterface) => {+    return queryInterface.sequelize.transaction(t => {+      return queryInterface.removeColumn('pipelines', 'datasetProcess', { transaction: t }).then(() => {

+'use strict';+import {QueryInterface, DataTypes} from 'sequelize';

+const os = require('os');++module.exports = {+  dialect: 'sqlite',+  storage: process.env.PIPCOOK_STORAGE || (`${os.homedir()}/.pipcook/db/pipcook.db`)+}

+const os = require('os');

   },   "scripts": {     "postinstall": "lerna bootstrap",-    "build": "lerna run compile",+    "build": "lerna run compile && rm ./packages/daemon/dist/migrations/*.d.ts",

 The `pipeline_name` is the name of the pipeline file under "test/pipelines", suc - "mnist-image-classification" - "databinding-image-classification" +### Database+If your modification involves `Database` structure, you need to write a migration script under `daemon/src/migrations`.

 The `pipeline_name` is the name of the pipeline file under "test/pipelines", suc - "mnist-image-classification" - "databinding-image-classification" +### Database+If your modification involves `Database` structure, you need to write a migration script under `daemon/src/migrations`.

+'use strict';++module.exports = {+  up: async (queryInterface, Sequelize) => {+    return queryInterface.sequelize.transaction(t => {+      return Promise.all([+        queryInterface.addColumn('pipelines', 'datasetProcess', {+          type: Sequelize.DataTypes.STRING+        }, { transaction: t }),+        queryInterface.addColumn('pipelines', 'datasetProcessParams', {+          type: Sequelize.DataTypes.STRING+        }, { transaction: t })+      ])+    })+  },++  down: async (queryInterface, Sequelize) => {+    return queryInterface.sequelize.transaction(t => {

+

++const os = require('os');++module.exports = {+  dialect: "sqlite",

     "ci": "npm run cov",     "compile": "midway-bin build -c --port=6927",     "clean": "rm -rf src/app/public",-    "test": "midway-bin test --ts --full-trace"+    "test": "midway-bin test --ts --full-trace",+    "migration": "npx sequelize-cli db:migrate",+    "undo-migration": "npx sequelize-cli db:migrate:undo"

+const path = require('path');

 And anyone could try your plugin via the following command: $ pipcook plugin install your-pipcook-plugin-name ``` +## Database++If your modification involves `Database` structure, you need to write a migration script under `daemon/src/migrations`.+For more detail about it, please refer to [Migration](https://sequelize.org/master/manual/migrations.html).

 function _internalWrap(T, src={}) {     invoke: {       enumerable: false,       writable: false,-      value: args => T.invoke.apply(T, args),+      value: args => {+        return T.invoke.apply(T, args);

+import { listen } from './request';+import { LogCallback } from './interface';++export class BaseApi {+  route: string;++  constructor(uri: string) {+    this.route = uri;+  }++  /**+   * listen log+   * @param logId log id+   * @param logCallback log callback+   */+  async log(logId: string, logCallback: LogCallback) {+    return new Promise((resolve, reject) => {+      listen(`${this.route}/log/${logId}`, undefined, {+        'log': (e: MessageEvent) => {+          const logObj = JSON.parse(e.data);+          if (logCallback) {+            logCallback(logObj.level, logObj.data);+          }+        },+        'error': (e: MessageEvent) => {+            reject(e.data);+        },+        'close': () => {

+import { listen } from './request';+import { LogCallback } from './interface';++export class BaseApi {+  route: string;++  constructor(uri: string) {+    this.route = uri;+  }++  /**+   * listen log+   * @param logId log id+   * @param logCallback log callback+   */+  async log(logId: string, logCallback: LogCallback) {+    return new Promise((resolve, reject) => {+      listen(`${this.route}/log/${logId}`, undefined, {+        'log': (e: MessageEvent) => {+          const logObj = JSON.parse(e.data);+          if (logCallback) {+            logCallback(logObj.level, logObj.data);+          }+        },+        'error': (e: MessageEvent) => {+            reject(e.data);+        },+        'close': () => {+          resolve();+        }+      });+    });+  }+}

   "name": "@pipcook/sdk",   "version": "1.0.4",   "description": "JavaScript SDK for accessing Pipcook Daemon",-  "main": "dist/index.js",+  "main": "src/index.ts",

 export class PipelineService {     }   } -  stopJob(id: string): boolean {+  stopJob(id: string): void {     const runnable = runnableMap[id];     if (runnable) {       runnable.destroy();       delete runnableMap[id];-      return true;     } else {-      return false;+      this.ctx.throw('no runnable found');

 export interface PluginResp { export interface PluginInstallingResp extends PluginResp {   logId: string; }++/**+ * response of pipline+ */+export interface PipelineResp {+  id: string;+  name: string;+  dataCollect: string;+  dataCollectParams: string;+  dataAccess: string;+  dataAccessParams: string;+  dataProcess: string;+  dataProcessParams: string;+  modelDefine: string;+  modelDefineParams: string;+  modelLoad: string;+  modelLoadParams: string;+  modelTrain: string;+  modelTrainParams: string;+  modelEvaluate: string;+  modelEvaluateParams: string;+}++/**+ * response of pipeline install+ */+export interface PipelineInstallingResp extends PipelineResp {+  logId: string;+}

 export class JobController {   @get('/:id')   public async get() {     const { id } = this.ctx.params;-    try {-      const job = await this.pipelineService.getJobById(id);-      if (!job) {-        throw new Error('job not found');-      }-      successRes(this.ctx, { data: job });-    } catch (err) {-      failRes(this.ctx, { message: err.message });+    const job = await this.pipelineService.getJobById(id);+    if (!job) {+      throw new Error('job not found');

-import { Context, controller, inject, provide, get, post } from 'midway';-import { successRes } from '../../utils/response';+import { controller, inject, provide, get, post, del, put } from 'midway';+import * as createHttpError from 'http-errors';+import { BaseController } from './base'; import { PluginManager } from '../../service/plugin'; import ServerSentEmitter from '../../utils/emitter';+import { PluginInstallingResp } from '../../interface'; import Debug from 'debug'; const debug = Debug('daemon.app.plugin');  @provide() @controller('/plugin')-export class PluginController {-  @inject()-  ctx: Context;+export class PluginController extends BaseController {    @inject('pluginManager')   pluginManager: PluginManager; -  @get('/install')+  // TODO(feely): check if the plugin has been installed+  /**+   * create plugin and install it+   */+  @post()   public async install() {+    const { name, pyIndex } = this.ctx.request.body;+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, false);+    this.success(response);+  }++  /**+   * reinstall plugin+   */+  @put()+  public async reinstall() {     const { name, pyIndex } = this.ctx.query;-    const sse = new ServerSentEmitter(this.ctx);-    try {-      debug(`checking info: ${name}.`);-      const pkg = await this.pluginManager.fetch(name);-      sse.emit('info', pkg);+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, true);+    this.success(response);+  } -      debug(`installing ${name}.`);-      const plugin = await this.pluginManager.findOrCreateByPkg(pkg);-      try {-        await this.pluginManager.install(pkg, pyIndex);-        sse.emit('installed', pkg);-      } catch (err) {-        await this.pluginManager.removeById(plugin.id);-        throw err;+  /**+   * delete plugin by name+   */+  @del('/:id')+  public async remove() {+    if (typeof this.ctx.params.id === 'string' && this.ctx.params.id) {+      const plugin = await this.pluginManager.findById(this.ctx.params.id);+      if (plugin) {+        await this.pluginManager.uninstall(plugin);+        this.success();+      } else {+        throw createHttpError(400, `no plugin found by id ${this.ctx.params.id}`);       }-    } catch (err) {-      sse.emit('error', err?.message);-    } finally {-      sse.finish();+    } else {+      throw createHttpError(400, 'no id value found');     }   }--  @get('/uninstall')-  public async uninstall() {-    await this.pluginManager.uninstall(this.ctx.query.name);-    successRes(this.ctx, {});+  /**+   * delete all plugins+   */+  @del()+  public async removeAll() {+    const plugins = await this.pluginManager.list();+    for (const plugin of plugins) {+      await this.pluginManager.uninstall(plugin);

 export class PluginManager {       throw err;     }   }+  async startInstall(pkg: PluginPackage, pyIndex?: string, force?: boolean): Promise<PluginInstallingResp> {

-import { Context, controller, inject, provide, get, post } from 'midway';-import { successRes } from '../../utils/response';+import { controller, inject, provide, get, post, del, put } from 'midway';+import * as createHttpError from 'http-errors';+import { BaseController } from './base'; import { PluginManager } from '../../service/plugin'; import ServerSentEmitter from '../../utils/emitter';+import { PluginInstallingResp } from '../../interface'; import Debug from 'debug'; const debug = Debug('daemon.app.plugin');  @provide() @controller('/plugin')-export class PluginController {-  @inject()-  ctx: Context;+export class PluginController extends BaseController {    @inject('pluginManager')   pluginManager: PluginManager; -  @get('/install')+  // TODO(feely): check if the plugin has been installed+  /**+   * create plugin and install it+   */+  @post()   public async install() {+    const { name, pyIndex } = this.ctx.request.body;+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, false);+    this.success(response);+  }++  /**+   * reinstall plugin+   */+  @put()+  public async reinstall() {     const { name, pyIndex } = this.ctx.query;-    const sse = new ServerSentEmitter(this.ctx);-    try {-      debug(`checking info: ${name}.`);-      const pkg = await this.pluginManager.fetch(name);-      sse.emit('info', pkg);+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, true);+    this.success(response);+  } -      debug(`installing ${name}.`);-      const plugin = await this.pluginManager.findOrCreateByPkg(pkg);-      try {-        await this.pluginManager.install(pkg, pyIndex);-        sse.emit('installed', pkg);-      } catch (err) {-        await this.pluginManager.removeById(plugin.id);-        throw err;+  /**+   * delete plugin by name+   */+  @del('/:id')+  public async remove() {+    if (typeof this.ctx.params.id === 'string' && this.ctx.params.id) {+      const plugin = await this.pluginManager.findById(this.ctx.params.id);+      if (plugin) {+        await this.pluginManager.uninstall(plugin);+        this.success();+      } else {+        throw createHttpError(400, `no plugin found by id ${this.ctx.params.id}`);       }-    } catch (err) {-      sse.emit('error', err?.message);-    } finally {-      sse.finish();+    } else {+      throw createHttpError(400, 'no id value found');     }   }--  @get('/uninstall')-  public async uninstall() {-    await this.pluginManager.uninstall(this.ctx.query.name);-    successRes(this.ctx, {});+  /**+   * delete all plugins+   */+  @del()+  public async removeAll() {+    const plugins = await this.pluginManager.list();+    for (const plugin of plugins) {+      await this.pluginManager.uninstall(plugin);+    }+    this.success();+  }+  /**+   * find a plugin by id+   */+  @get('/:id')+  public async get() {+    const plugin = await this.pluginManager.findById(this.ctx.params.id);+    if (plugin) {+      this.success(plugin);+    } else {+      throw createHttpError(404, 'no plugin found');+    }   } -  @get('/list')+  /**+   * list plugins+   */+  @get()   public async list() {     const plugins = await this.pluginManager.list({       datatype: this.ctx.query.datatype,       category: this.ctx.query.category     });-    successRes(this.ctx, { data: plugins });+    this.success(plugins);   }-  @get('/:id')-  public async info() {-    const plugin = await this.pluginManager.findById(this.ctx.params.id);-    successRes(this.ctx, { data: plugin });-  }-  @post('/upload')-  public async upload() {++  /**+   * create a plugin by tarball stream+   */+  @post('/tarball')+  public async uploadPackage() {     const fs = await this.ctx.getFileStream();

-import { Context, controller, inject, provide, get, post } from 'midway';-import { successRes } from '../../utils/response';+import { controller, inject, provide, get, post, del, put } from 'midway';+import * as createHttpError from 'http-errors';+import { BaseController } from './base'; import { PluginManager } from '../../service/plugin'; import ServerSentEmitter from '../../utils/emitter';+import { PluginInstallingResp } from '../../interface'; import Debug from 'debug'; const debug = Debug('daemon.app.plugin');  @provide() @controller('/plugin')-export class PluginController {-  @inject()-  ctx: Context;+export class PluginController extends BaseController {    @inject('pluginManager')   pluginManager: PluginManager; -  @get('/install')+  // TODO(feely): check if the plugin has been installed+  /**+   * create plugin and install it+   */+  @post()   public async install() {+    const { name, pyIndex } = this.ctx.request.body;+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, false);+    this.success(response);+  }++  /**+   * reinstall plugin+   */+  @put()+  public async reinstall() {     const { name, pyIndex } = this.ctx.query;-    const sse = new ServerSentEmitter(this.ctx);-    try {-      debug(`checking info: ${name}.`);-      const pkg = await this.pluginManager.fetch(name);-      sse.emit('info', pkg);+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, true);+    this.success(response);+  } -      debug(`installing ${name}.`);-      const plugin = await this.pluginManager.findOrCreateByPkg(pkg);-      try {-        await this.pluginManager.install(pkg, pyIndex);-        sse.emit('installed', pkg);-      } catch (err) {-        await this.pluginManager.removeById(plugin.id);-        throw err;+  /**+   * delete plugin by name+   */+  @del('/:id')+  public async remove() {+    if (typeof this.ctx.params.id === 'string' && this.ctx.params.id) {+      const plugin = await this.pluginManager.findById(this.ctx.params.id);+      if (plugin) {+        await this.pluginManager.uninstall(plugin);+        this.success();+      } else {+        throw createHttpError(400, `no plugin found by id ${this.ctx.params.id}`);       }-    } catch (err) {-      sse.emit('error', err?.message);-    } finally {-      sse.finish();+    } else {+      throw createHttpError(400, 'no id value found');     }   }--  @get('/uninstall')-  public async uninstall() {-    await this.pluginManager.uninstall(this.ctx.query.name);-    successRes(this.ctx, {});+  /**+   * delete all plugins+   */+  @del()+  public async removeAll() {+    const plugins = await this.pluginManager.list();+    for (const plugin of plugins) {+      await this.pluginManager.uninstall(plugin);

-import { Context, controller, inject, provide, get, post } from 'midway';-import { successRes } from '../../utils/response';+import { controller, inject, provide, get, post, del, put } from 'midway';+import * as createHttpError from 'http-errors';+import { BaseController } from './base'; import { PluginManager } from '../../service/plugin'; import ServerSentEmitter from '../../utils/emitter';+import { PluginInstallingResp } from '../../interface'; import Debug from 'debug'; const debug = Debug('daemon.app.plugin');  @provide() @controller('/plugin')-export class PluginController {-  @inject()-  ctx: Context;+export class PluginController extends BaseController {    @inject('pluginManager')   pluginManager: PluginManager; -  @get('/install')+  // TODO(feely): check if the plugin has been installed+  /**+   * create plugin and install it+   */+  @post()   public async install() {+    const { name, pyIndex } = this.ctx.request.body;+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, false);+    this.success(response);+  }++  /**+   * reinstall plugin+   */+  @put()+  public async reinstall() {     const { name, pyIndex } = this.ctx.query;-    const sse = new ServerSentEmitter(this.ctx);-    try {-      debug(`checking info: ${name}.`);-      const pkg = await this.pluginManager.fetch(name);-      sse.emit('info', pkg);+    let response: PluginInstallingResp;+    debug(`checking info: ${name}.`);+    response = await this.pluginManager.installByName(name, pyIndex, true);