From 8b222389849734601222c4b0daa2382538cb6229 Mon Sep 17 00:00:00 2001 From: KonghaYao <20192831006@m.scnu.edu.cn> Date: Mon, 2 Aug 2021 10:01:56 +0800 Subject: [PATCH] =?UTF-8?q?feat(task):=20=E6=96=B0=E7=9A=84=20Task=20?= =?UTF-8?q?=E5=87=86=E5=A4=87=E6=9B=BF=E6=8D=A2=E6=97=A7=E7=9A=84=20Task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Design.md => Design.md | 0 TODO.md | 14 +++ VERSION.md | 6 +- src/index.js => index.js | 10 +- jspider.drawio | 160 +++++++++++++++++++++++++++++++ package.json | 2 + pnpm-lock.yaml | 16 ++++ rollup.config.dev.js | 9 +- src/ControlPanel/ControlPanel.js | 4 +- src/TaskSystem/StaticEvent.js | 14 +-- src/jspider.drawio | 123 ------------------------ test/TaskState/Task.js | 40 ++++++++ test/TaskState/TaskProps.js | 21 ++++ test/TaskState/TaskState.js | 65 +++++++++++++ test/TaskState/TaskTypes.js | 22 +++++ test/mobx-test.mjs | 85 ++++++++++++++++ test/mobx.state.tree.mjs | 41 ++++++++ test/modules.json | 2 +- test/test-deno.js | 12 --- 19 files changed, 489 insertions(+), 157 deletions(-) rename src/Design.md => Design.md (100%) create mode 100644 TODO.md rename src/index.js => index.js (51%) create mode 100644 jspider.drawio delete mode 100644 src/jspider.drawio create mode 100644 test/TaskState/Task.js create mode 100644 test/TaskState/TaskProps.js create mode 100644 test/TaskState/TaskState.js create mode 100644 test/TaskState/TaskTypes.js create mode 100644 test/mobx-test.mjs create mode 100644 test/mobx.state.tree.mjs delete mode 100644 test/test-deno.js diff --git a/src/Design.md b/Design.md similarity index 100% rename from src/Design.md rename to Design.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..097ee3b --- /dev/null +++ b/TODO.md @@ -0,0 +1,14 @@ +# JSpider 项目看板 + +JSpider 项目看板中主要有很多关于 JSpider 的未来目标 + +### Todo + +- [ ] + +### In Progress + + +### Done ✓ + + diff --git a/VERSION.md b/VERSION.md index a960ecd..199a535 100644 --- a/VERSION.md +++ b/VERSION.md @@ -10,9 +10,9 @@ - 完整的 JSpider 编程逻辑框架出图 - - [ ] JSpider 中的数据流系统 + - [x] JSpider 中的数据流系统 - - [ ] JSpider 中的事件系统 + - [x] JSpider 中的事件系统 - [ ] JSpider 需要实现 MVVM 模型来实现爬虫系统的可视化 @@ -20,7 +20,7 @@ - [x] Control Panel 暂停功能 - - [ ] Control Panel 重试功能（非流内自动重试） + - [x] Control Panel 重试功能（非流内自动重试） - 重构 plugins - [ ] plugins 分包导入 diff --git a/src/index.js b/index.js similarity index 51% rename from src/index.js rename to index.js index 884acf6..ef24288 100644 --- a/src/index.js +++ b/index.js @@ -3,11 +3,11 @@ * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ -import { Spider } from './Spider/index'; -import * as plugins from '../plugins/index.js'; -import * as tools from '../tools/index.js'; // 工具都是 $ 开头的函数 -import { Plugin } from './Pipeline/PluginSystem'; -import { Task, TaskGroup } from './TaskSystem/index'; +import { Spider } from '@src/Spider/index'; +import * as plugins from '@plugins/index.js'; +import * as tools from '@tools/index.js'; // 工具都是 $ 开头的函数 +import { Plugin } from '@src/Pipeline/PluginSystem'; +import { Task, TaskGroup } from '@src/TaskSystem/index'; export default Object.assign(Spider, tools, { plugins, Plugin, diff --git a/jspider.drawio b/jspider.drawio new file mode 100644 index 0000000..b7f8ea6 --- /dev/null +++ b/jspider.drawio @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/package.json b/package.json index 58eb194..bce645d 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,8 @@ "chalk": "^4.1.1", "consola": "^2.15.3", "lodash-es": "^4.17.21", + "mobx": "^6.3.2", + "mobx-state-tree": "^5.0.2", "mockjs": "^1.1.0", "rxjs": "6.6.6", "uuid": "^8.3.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5efbbe3..8ef3068 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,8 @@ specifiers: eslint-plugin-prettier: ^3.4.0 lodash-es: ^4.17.21 mitt: ^3.0.0 + mobx: ^6.3.2 + mobx-state-tree: ^5.0.2 mockjs: ^1.1.0 prettier: ^2.3.2 rollup: ^2.50.5 @@ -38,6 +40,8 @@ dependencies: chalk: 4.1.1 consola: 2.15.3 lodash-es: 4.17.21 + mobx: 6.3.2 + mobx-state-tree: 5.0.2_mobx@6.3.2 mockjs: 1.1.0 rxjs: 6.6.6 uuid: 8.3.2 @@ -3277,6 +3281,18 @@ packages: hasBin: true dev: true + /mobx-state-tree/5.0.2_mobx@6.3.2: + resolution: {integrity: sha512-hQ2Ewbf2hwB4BNAq0ysYMXywbXQ4bHMpAJRnquzKtUBY1OclZTMBg1Dc2gkZNWfU9PrJ03s2tHK5X5c8wtHAHA==} + peerDependencies: + mobx: ^6.3.0 + dependencies: + mobx: 6.3.2 + dev: false + + /mobx/6.3.2: + resolution: {integrity: sha512-xGPM9dIE1qkK9Nrhevp0gzpsmELKU4MFUJRORW/jqxVFIHHWIoQrjDjL8vkwoJYY3C2CeVJqgvl38hgKTalTWg==} + dev: false + /mockjs/1.1.0: resolution: {integrity: sha512-eQsKcWzIaZzEZ07NuEyO4Nw65g0hdWAyurVol1IPl1gahRwY+svqzfgfey8U8dahLwG44d6/RwEzuK52rSa/JQ==} hasBin: true diff --git a/rollup.config.dev.js b/rollup.config.dev.js index 7280d75..359bb0e 100644 --- a/rollup.config.dev.js +++ b/rollup.config.dev.js @@ -39,10 +39,11 @@ export default { resolve({ browser: true, }), - // babel({ - // exclude: 'node_modules/**', // 仅仅转译我们的源码 - // include: 'src', - // }), + babel({ + exclude: 'node_modules/**', // 仅仅转译我们的源码 + include: 'src', + babelHelpers: 'bundled', + }), commonjs(), // 将 CommonJS 转换成 ES2015 模块供 Rollup 处理 livereload({ watch: 'dist' }), serve({ diff --git a/src/ControlPanel/ControlPanel.js b/src/ControlPanel/ControlPanel.js index 1f89742..74cabff 100644 --- a/src/ControlPanel/ControlPanel.js +++ b/src/ControlPanel/ControlPanel.js @@ -16,7 +16,7 @@ import { pauseToggle } from '../utils/pauseToggle'; export class ControlPanel { state = 'free'; // 'free' 'preparing' - #runningQueue = new functionQueue(); // init 阶段的 Queue 队列 + #runningQueue = new functionQueue(); // 异步 Queue 队列 _stop = false; // 用于直接切断 spiderSource$ 的流 spiderSource$ = null; _pipeline = null; @@ -61,7 +61,7 @@ export class ControlPanel { } } - // startInfo --TaskManager.createTask--> Task --emit--EventHub--> Flow + // |startInfo| - TaskManager.createTask -> |Task| - emit EventHub -> |Flow| createFlow(infos) { return this.#runningQueue.enQueue(() => { infos.forEach((info) => { diff --git a/src/TaskSystem/StaticEvent.js b/src/TaskSystem/StaticEvent.js index e5ebbbc..d451043 100644 --- a/src/TaskSystem/StaticEvent.js +++ b/src/TaskSystem/StaticEvent.js @@ -7,24 +7,24 @@ // 这些事件一般通过 $commit(eventName,payload) 进行执行 // ! this 被绑定为 Task 的实例 export const staticEvent = { - start(UUID) { + start(uuid) { this._status = 'busy'; // ! 这里会有一个清空这个 key 的 value 的情况，这是因为 start 是强制的 - this._progress.set(UUID, { process: 'start' }); + this._progress.set(uuid, { uuid, process: 'start' }); return this._output || this._originData; }, - success(output, UUID) { + success(output, uuid) { this._status = 'free'; - this._progress.set(UUID, { process: 'success', output }); + this._progress.set(uuid, { uuid, process: 'success', output }); this._output = output; }, - complete(UUID) { + complete(uuid) { this._status = 'complete'; }, - error(err, UUID) { + error(err, uuid) { this._status = 'error'; - this._progress.set(UUID, { process: 'error', err }); + this._progress.set(uuid, { uuid, process: 'error', err }); }, destroy() { this._process = []; diff --git a/src/jspider.drawio b/src/jspider.drawio deleted file mode 100644 index 37ec5b7..0000000 --- a/src/jspider.drawio +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test/TaskState/Task.js b/test/TaskState/Task.js new file mode 100644 index 0000000..5243582 --- /dev/null +++ b/test/TaskState/Task.js @@ -0,0 +1,40 @@ +import { staticEvent } from './TaskEvent.js'; +import { createTaskStore } from './TaskState.js'; +export class Task { + _belongTo = null; // 当有 TaskGroup 时，指向 Group + constructor(_originData, _spiderUUID) { + this.$EventHub = new EventHub(staticEvent, this); + + // 由 store 验证相关的正确性 + this.$store = createTaskStore({ spiderUUID, originData }); + } + + // Plugin 的汇报口 + $commit(type, ...payload) { + const result = this.$store[type](...payload); + this.$EventHub.emit(type, ...payload); + return result; + } + + // 外部系统的监控口 + $on(...args) { + return this.$EventHub.on(...args); + } + $off(...args) { + return this.$EventHub.off(...args); + } + $isSameTask(task) { + return task.$store.spiderUUID === this.$store.spiderUUID && task.$store.uuid === this.$store.uuid; + } + $checkRepeat(uuid) { + return this.$store.dataSlide.includes(uuid); + } + $destroy() { + this._belongTo = null; + this.$off('*'); + this.$commit('destroy'); // 通知外部，该 Task 被销毁 + } + get [Symbol.toStringTag]() { + return 'Task'; + } +} diff --git a/test/TaskState/TaskProps.js b/test/TaskState/TaskProps.js new file mode 100644 index 0000000..2f45789 --- /dev/null +++ b/test/TaskState/TaskProps.js @@ -0,0 +1,21 @@ +/** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ +import { v4 as uuidv4 } from 'uuid'; +export function createProps({ spiderUUID, originData }) { + // 创建初始化的参数 + return { + uuid: uuidv4(), + spiderUUID, // JSpider 的实例的 UUID + createdAt: new Date(), + error: '', + status: 'free', // 这个位置是为了让 Plugin 能识别的一个标识 + updatedAt: new Date(), + dataSlideUUID: spiderUUID, + dataSlide: [], + originData, + output: null, // 每个中间件传出的数据 + }; +} diff --git a/test/TaskState/TaskState.js b/test/TaskState/TaskState.js new file mode 100644 index 0000000..43f0aac --- /dev/null +++ b/test/TaskState/TaskState.js @@ -0,0 +1,65 @@ +/** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ + +import { types, getSnapshot, applySnapshot, destroy } from 'mobx-state-tree'; +import { createProps } from './TaskProps.js'; +import { AutoType, TaskState } from './TaskTypes.js'; + +const TaskStore = types + .model({ + uuid: types.string, + spiderUUID: types.string, // JSpider 的实例的 UUID + dataSlide: types.array(type.string), // 更改为存储 UUID + dataSlideUUID: types.string, // 上一次完成的 UUID + createdAt: types.Date, + error: types.maybe(type.string), + status: TaskState, // 这个位置是为了让 Plugin 能识别的一个标识 + updatedAt: types.Date, + // 记录完成过的 uuid 的信息，只有在 StaticEvent.js 中才能更改 + originData: types.maybe(AutoType.anyType), + + // 每个中间件传出的数据 + output: types.maybe(AutoType.anyType), + }) + .actions((self) => { + return { + $backup() { + return getSnapshot(self); + }, + $import(backup) { + return applySnapshot(backup); + }, + + start() { + self.status = 'pending'; + return self.output || self.originData; + }, + success(output, uuid) { + self.status = 'free'; + self.output = output; + this._addUUIDToDataSlide(uuid); + }, + complete(uuid) { + self.status = 'complete'; + this._addUUIDToDataSlide(uuid); + }, + error(err, uuid) { + self.status = 'error'; + self.error = err; + this._addUUIDToDataSlide(uuid); + }, + destroy() { + destroy(self); + }, + _addUUIDToDataSlide(uuid) { + self.dataSlide.push(uuid); + self.dataSlideUUID = uuid; + }, + }; + }); +export function createTaskStore(model) { + return TaskStore.create(createProps(model)); +} diff --git a/test/TaskState/TaskTypes.js b/test/TaskState/TaskTypes.js new file mode 100644 index 0000000..8dd436e --- /dev/null +++ b/test/TaskState/TaskTypes.js @@ -0,0 +1,22 @@ +import { types } from 'mobx-state-tree'; +const TaskState = types.enumeration('TaskState', ['free', 'pending', 'complete', 'error', 'destroyed']); + +const AutoType = { + anyType: types.custom({ + name: 'any', + fromSnapshot(value) { + return value; + }, + toSnapshot(value) { + return value; + }, + isTargetType() { + return true; + }, + getValidationMessage() { + return ''; + }, + }), +}; + +export { AutoType, TaskState }; diff --git a/test/mobx-test.mjs b/test/mobx-test.mjs new file mode 100644 index 0000000..21ff694 --- /dev/null +++ b/test/mobx-test.mjs @@ -0,0 +1,85 @@ +import { types, getSnapshot, applySnapshot } from 'mobx-state-tree'; + +const TaskState = types.enumeration('TaskState', ['free', 'pending', 'complete', 'error', 'destroyed']); + +const pluginOutput = types.custom({ + name: 'pluginOutput', + fromSnapshot(value) { + return value; + }, + toSnapshot(value) { + return value; + }, + isTargetType(value) { + return true; + }, + getValidationMessage(value) { + return ''; + }, +}); +const anyType = types.custom({ + name: 'any', + fromSnapshot(value) { + return value; + }, + toSnapshot(value) { + return value; + }, + isTargetType() { + return true; + }, + getValidationMessage() { + return ''; + }, +}); +function init(_spiderUUID) { + return { + _uuid: '', + _spiderUUID, // JSpider 的实例的 UUID + _createdAt: new Date(), + + _status: 'free', // 这个位置是为了让 Plugin 能识别的一个标识 + _updatedAt: new Date(), + _progress: { + a: { + uuid: '347847328', + state: 'free', + err: undefined, + output: null, + }, + }, // 记录完成过的 uuid 的信息，只有在 StaticEvent.js 中才能更改 + _originData: null, + _output: null, // 每个中间件传出的数据 + }; +} +const PluginResult = types.model({ + uuid: types.string, + state: TaskState, + err: types.maybe(types.string), + output: types.maybe(pluginOutput), +}); +const Data = types + .model({ + _uuid: types.string, + _spiderUUID: types.string, // JSpider 的实例的 UUID + _createdAt: types.Date, + + _status: 'free', // 这个位置是为了让 Plugin 能识别的一个标识 + _updatedAt: types.Date, + _progress: types.map(PluginResult), // 记录完成过的 uuid 的信息，只有在 StaticEvent.js 中才能更改 + _originData: types.maybe(anyType), + _output: types.maybe(anyType), // 每个中间件传出的数据 + }) + .actions((self) => { + return { + $backup() { + return getSnapshot(self); + }, + $import(backup) { + return applySnapshot(backup); + }, + }; + }); +const store = Data.create({ a: 1, ...init('123232') }); +console.log('开始'); +console.log(store); diff --git a/test/mobx.state.tree.mjs b/test/mobx.state.tree.mjs new file mode 100644 index 0000000..64f15b1 --- /dev/null +++ b/test/mobx.state.tree.mjs @@ -0,0 +1,41 @@ +import { types, getSnapshot, applySnapshot } from 'mobx-state-tree'; +// declaring the shape of a node with the type `Todo` +const Todo = types.model({ + title: types.string, + done: types.optional(types.boolean, false), +}); + +const TodoStore = types + .model('TodoStore', { + todos: types.array(Todo), + selectedTodo: types.reference(Todo), + }) + .views((self) => { + return { + get completedTodos() { + return self.todos.filter((t) => t.done); + }, + findTodosByUser(user) { + return self.todos.filter((t) => t.user === user); + }, + }; + }) + .actions((self) => { + return { + addTodo(title) { + self.todos.push({ + id: Math.random(), + title, + }); + }, + }; + }); +const todo = Todo.create({ + title: '100', +}); +const storeInstance = TodoStore.create({ + selectedTodo: 0, + todos: [todo], +}); +console.log(getSnapshot(storeInstance)); +console.log(storeInstance.selectedTodo); diff --git a/test/modules.json b/test/modules.json index 893e80c..6bc9e5e 100644 --- a/test/modules.json +++ b/test/modules.json @@ -1,6 +1,6 @@ { "imports": { - "mobx": "https://ga.jspm.io/npm:mobx@6.3.2/dist/mobx.esm.js", + "mobx": "https://ga.jspm.io/npm:mobx@6.3.2/dist/mobx.esm.production.min.js", "lodash": "https://ga.jspm.io/npm:lodash@4.17.21/lodash.js", "mitt": "https://ga.jspm.io/npm:mitt@3.0.0/dist/mitt.mjs", "redux": "https://ga.jspm.io/npm:redux@4.1.0/lib/dev.redux.js", diff --git a/test/test-deno.js b/test/test-deno.js deleted file mode 100644 index 38ab1cf..0000000 --- a/test/test-deno.js +++ /dev/null @@ -1,12 +0,0 @@ -@log -class MyClass { - logger = null; -} - -function log(target) { - // 这个 target 在这里就是 MyClass 这个类 - target.prototype.logger = () => `${target.name} 被调用`; -} - -const test = new MyClass(); -test.logger(); // MyClass 被调用