diff --git a/.eslintignore b/.eslintignore index d1346c8..1b7c014 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,2 +1,3 @@ **/node_modules/* **/dist/* +**/lib/* diff --git a/.prettierignore b/.prettierignore index d1346c8..1b7c014 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,3 @@ **/node_modules/* **/dist/* +**/lib/* diff --git a/Design.md b/Design.md index d56ff51..0fa3074 100644 --- a/Design.md +++ b/Design.md @@ -2,57 +2,47 @@ _这是一个关于 JSpider 的结构设计文件_ -## System 总体结构 +## 总体结构 **_下面的部分都称为 System,是 JSpider 流程中的一环,但是为了方便开发者使用,这些系统有可能不会被开发者看到,或者是以其他形式被开发者所使用。_** -### 1 Spider +### 1. Task -**_Spider_** 是 JSpider 的爬虫流程的外部 API,是 ControlPanel 的 Console 版本。 +> **Task** 是独立的最小单元,是初始信息(originData)的封装。 -职能: +**_Task_** 是 JSpider 的 Data flow 中封装的最小单位。也就是对数据的封装的一层,以便有统一可以调用的 API。 -1. 主要是为用户交互提供方式 +**_TaskGroup_** 是 Task 的合成型,在遇到某些 plugin 的时候需要使用。 -### 2. Task +同时,在 **Task** 的组成中使用了 **mobx-state-tree** 和 **EventBus** 进行信息更新的发布和传播。 -**_Task_** 是 JSpider 的 Data flow 中封装的最小单位。也就是对数据封装一层,以便有可以调用的 API。 +### 2. Pipeline -### 3. Pipeline +> **Task** 像是水滴,流动在水管(**Pipeline**)中,最后流出。 **_Pipeline_** 是 JSpider 中用于将各种操作合成为一个函数步骤的快捷方式,本质为一个 rxjs operator 的生成工厂。 -职能: +**_Pipeline_** 通过将 **Task** 传递给 plugin 来实现数据的迭代处理。 -1. 实现 Request - -2. 实现各种各样的数据类型转换,如 JSON 转 xlsx Blob - -3. 删减无用数据项 - -### 4. ControlPanel +### 3. ControlPanel **_ControlPanel_** 是用于调控数据的流量的一个总 system,全局唯一,主要担当中央控制器的作用。 -### 5. Memory +**_ControlPanel_** 负责处理用户输入的信息,**_Mirror_** 负责信息的传播 -**_Memory_** 是 JSpider 用来管理数据的方式,可以认为是 JSpider 和数据库的中间人,所有的数据通过 JSpider 中的 **_Memory_** 来间接修改数据库中的数据。 +### 4. Mirror -职能: +> **镜子(Mirror)只是反射光** -1. 在传入数据的时候,进行相应的 **数据库的调度**。 +**_Mirror_** 只是对数据更新进行传播,保证可以渲染到指定的 View 中。 -2. 在需要的时候执行 **下载文件** 的功能。 +### 5. View -### 5. Fake Server +View 是从 Mirror 订阅过来的数据的自定义接口,可供开发者自行进行信息的订阅,方便进行 View 的扩充。 -**_Fake Server_** 是 JSpider 中的 ajax 拦截代理系统。 +### 6. Fake Server -职能: - -1. 通过制造 **虚假数据** 来迷惑前端的函数执行相应的步骤, - -2. 创建一个 **虚拟服务器** 给前端的爬虫学习者提供相应的练习,这样就可以直接在网站上进行爬虫练习而不用后端服务器了。 +**_Fake Server_** 是 JSpider 中的 ajax 拦截代理系统。 ## Plugin 机制 @@ -62,7 +52,7 @@ _这是一个关于 JSpider 的结构设计文件_ ## 项目的代码结构 -### 1. src 主要代码 +### 1. src 为主要代码 ### 2. package 文件夹下的分代码库 diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 097ee3b..0000000 --- a/TODO.md +++ /dev/null @@ -1,14 +0,0 @@ -# JSpider 项目看板 - -JSpider 项目看板中主要有很多关于 JSpider 的未来目标 - -### Todo - -- [ ] - -### In Progress - - -### Done ✓ - - diff --git a/VERSION.md b/VERSION.md index e0172c0..4d2e019 100644 --- a/VERSION.md +++ b/VERSION.md @@ -5,24 +5,23 @@ ### 3.3 版本更新 - [ ] 关于 UI 界面的更新 +- [ ] FakeServer 的载入 +- 重构 plugins + - [ ] plugins 分包导入 ### 3.2 - Paimon 版本更新 +> 派蒙版本,即最初的伙伴。这个版本的编程基础完成,可以在 Console 中对 jspider 进行交互操作。但是 plugin 并未完全移植成功,尚需努力。(。>︿<)\_θ + - 完整的 JSpider 编程逻辑框架出图 - [x] JSpider 中的数据流系统 - - [x] JSpider 中的事件系统 - - [x] JSpider 需要实现 MVVM 模型来实现爬虫系统的可视化 + - 这个使用了 Mobx 进行数据监听,rxjs 进行数据传播,3.3 将会以 Vue 作为可视化的 View 层。 - 重构所有的核心代码 - - [x] Control Panel 暂停功能 - - [x] Control Panel 重试功能(非流内自动重试) -- 重构 plugins - - [x] plugins 分包导入 - -> 因为是 3.2 版本才更新这个文件,所以以前的版本都丢失了。 +### 因为是 3.2 版本才更新这个文件,所以往前的版本都丢失了。 diff --git a/dist/JSpider.esm.min.js b/dist/JSpider.esm.min.js index a1c4420..1aa77f6 100644 --- a/dist/JSpider.esm.min.js +++ b/dist/JSpider.esm.min.js @@ -13564,6 +13564,9 @@ const TaskStore = types }, start() { + if (self.status === 'pending') throw new Error('Task 处在 pending 状态'); + self.dataSlide = []; + self.dataSlideUUID = ''; self.status = 'pending'; return self.output || self.originData; }, @@ -16429,9 +16432,7 @@ class Task { $isSameTask(task) { return task.$store.spiderUUID === this.$store.spiderUUID && task.$store.uuid === this.$store.uuid; } - $checkRepeat(uuid) { - return this.$store.dataSlide.includes(uuid); - } + $destroy() { this._belongTo = null; this.$commit('destroy'); // 先通知外部,该 Task 被销毁 @@ -16490,17 +16491,60 @@ class TaskGroup$1 extends Task { * SPDX-License-Identifier: Apache-2.0 */ +/** + * Mirror 是 JSpider 中的数据外放接口 + * + * MessageHub 是集合所有事件的事件中心, + * 当 MessageHub 被 emit 时, MessageHub 发出相关的 Update 进行视图的更新 + * 所有的外放接口使用 rxjs 进行 subscribe + */ + +const MessageHub = new EventHub(); + +// 任何一个 Task 数据发生改变时 +const TaskUpdate = MessageHub.createSource$('TaskUpdate'); + +// ControlPanel 的状态发生改变 +/** + * ControlUpdate + * + * MessageHub.emit('ControlUpdate', payload); + * payload: { + * name:'your message token',// stateChange flowStart flowStop + * value:any, + * } + * + */ + +const ControlUpdate = MessageHub.createSource$('ControlUpdate'); + +/** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ var staticEvent = { // ! 即使这些函数不会被使用,也必须使用注释的方式写下 stateChange(state) { - this.state = state; + this.status = state; + MessageHub.emit('ControlUpdate', { + name: 'stateChange', + value: state, + }); }, 'Flow:stop'() { this._stop = true; + MessageHub.emit('ControlUpdate', { + name: 'flowStop', + }); }, 'Flow:start'() { - console.log('start'); + console.log('jspider 开始运行'); + this.status = 'pending'; this._stop = false; + MessageHub.emit('ControlUpdate', { + name: 'flowStart', + }); }, // 'Flow:input'() {} @@ -16510,11 +16554,16 @@ var staticEvent = { if (task instanceof TaskGroup$1) { task.$destroy(); } + MessageHub.emit('ControlUpdate', { + name: 'TaskSuccess', + }); }, 'Task:error'(error) { console.log(error); }, - 'Task:complete'() {}, + // 'Flow:complete'() { + // console.log('线路完成'); + // }, }; /** @@ -16522,26 +16571,11 @@ var staticEvent = { * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ +function bindUpdate() { + const backup = this.$store.$backup(); -/** - * Mirror 是 JSpider 中的数据外放接口 - * - * MessageHub 是集合所有事件的事件中心, - * 当 MessageHub 被 emit 时, MessageHub 发出相关的 Update 进行视图的更新 - * 所有的外放接口使用 rxjs 进行 subscribe - */ - -const MessageHub = new EventHub(); - -// Task 数据发生改变时 -const TaskUpdate = MessageHub.createSource$('TaskUpdate'); - -/** - * @license - * Copyright 2021 KonghaYao 江夏尧 - * SPDX-License-Identifier: Apache-2.0 - */ - + MessageHub.emit('TaskUpdate', backup); +} // ! 用于维护全局 Task 数据的中心 class TaskManager { #Tasks = new Map(); // 用于维护所有的 Task @@ -16554,18 +16588,16 @@ class TaskManager { this.#Tasks.set(task.uuid, task); const that = this; task.$on({ + start: bindUpdate, + success: bindUpdate, + complete: bindUpdate, + error: bindUpdate, // 监听事件,并更新响应的 viewModel destroy() { that.#Tasks.delete(this.uuid); // this 绑定的是 task }, }); - ['start', 'success', 'complete', 'error'].forEach((name) => { - task.$on(name, function () { - const backup = this.$store.$backup(); - MessageHub.emit('TaskUpdate', backup); - }); - }); this.viewModel.push(task.$store); return task; } @@ -16625,14 +16657,13 @@ function pauseToggle(openings, closings) { * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ - // ControlPanel 是 JSpider 内部的事件和数据中心。 // 全部 JSpider 涉及到的边界中,ControlPanel 只有一个,但是 View 可以有多个,而 Spider 就是 View 中的一个 // 用于分发数据流,提供 Task 的状态变更。 // TODO 并且可以提供数据的响应给类似于 UI 界面形成可视化 class ControlPanel$1 { - state = 'free'; // 'free' 'preparing' + status = 'free'; // 'free' 'preparing' #runningQueue = new functionQueue(); // 异步 Queue 队列 _stop = false; // 用于直接切断 spiderSource$ 的流 spiderSource$ = null; @@ -16656,13 +16687,13 @@ class ControlPanel$1 { // 所有的事件分配到 staticEvent 中去写 (task) => this.$EventHub.emit('Task:success', task), (error) => this.$EventHub.emit('Task:error', error), - () => this.$EventHub.emit('Task:complete'), + () => this.$EventHub.emit('Flow:complete'), ); this.$EventHub.emit('Flow:stop'); } set pipeline(value) { - if (this.state === 'free') { + if (this.status === 'free') { this.$EventHub.emit('stateChange', 'preparing'); this._pipeline = value; this.#runningQueue.enQueue( @@ -16709,13 +16740,36 @@ class ControlPanel$1 { var ControlPanel = new ControlPanel$1(); -/** - * @license - * Copyright 2021 KonghaYao 江夏尧 - * SPDX-License-Identifier: Apache-2.0 - */ +// 注意,这里只要 View 被实例化了,它就会订阅数据 class View { - constructor() {} + constructor({ tasks = false, controlPanel = false } = {}) { + this.config = { + tasks, + controlPanel, + }; + + if (tasks) { + const a = TaskUpdate.subscribe((data) => this._update(data)); + this.#subscriber.push(a); + } + if (controlPanel) { + const b = ControlUpdate.subscribe((data) => this._change(data)); + this.#subscriber.push(b); + } + } + config = {}; + #subscriber = []; + $destroy() { + this.#subscriber.forEach((sub) => { + sub.unsubscribe(); + }); + } +} + +class ConsoleView extends View { + constructor(config) { + super(Object.assign(config, { tasks: true, controlPanel: true })); + } tasks = []; #uuidArray = []; _update(data) { @@ -16728,20 +16782,22 @@ class View { this.tasks.splice(index, 1, data); } } + _change({ name, value = '' }) {} } -// Spider 是 Console 的数据放送 + +/** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ +// Spider 是一个 View class Spider { constructor({ logEvery = false } = {}) { this.config = { logEvery, }; - if (logEvery) { - TaskUpdate.subscribe((data) => { - this.views._update(data); - }); - } + this.views = new ConsoleView(this.config); } - views = new View(); crawl(...args) { ControlPanel.createFlow(args.flat()); return this; @@ -16803,7 +16859,7 @@ class PLUGIN$1 { name, // 名称,一般用作提示标记 main, // Plugin 中的功能性函数 init, // 初始化整个 Plugin 的函数 - error, // 函数错误时的事件 + error, // operator 错误时的事件,若返回 false 类的数据,将会中断流,返回正常数据会继续流 complete, // 函数完成时的提示事件 options, // main 函数接收的 options saveResult, // 是否保存结果到每一个 Task 中 @@ -16818,24 +16874,18 @@ class PLUGIN$1 { return of(task).pipe( // 设置跳过 Plugin 的逻辑 switchMap((task) => { - if (task.$checkRepeat(this.uuid) || this.forceRetry) { - return of(task).pipe( - map$1((task) => [task.$commit('start', this.uuid), task._originData]), - - switchMap(([data, originData]) => { - const result = this.main(data, originData); - return result instanceof Promise || result instanceof Observable - ? from(result) - : of(result); - }), - map$1((result) => { - task.$commit('success', result, this.uuid, this.saveResult); - return task; - }), - ); - } - console.log('跳过一个目标'); - return of(task); + return of(task).pipe( + map$1((task) => [task.$commit('start', this.uuid), task._originData]), + + switchMap(([data, originData]) => { + const result = this.main(data, originData); + return result instanceof Promise || result instanceof Observable ? from(result) : of(result); + }), + map$1((result) => { + task.$commit('success', result, this.uuid, this.saveResult); + return task; + }), + ); }), // 捕获到异常 catchError((...args) => { @@ -16859,11 +16909,13 @@ class PLUGIN$1 { } } -function Plugin$1(Process) { +function Plugin$1(Process, otherOptions = {}) { if (Process instanceof Function) { - return new PLUGIN$1({ - main: Process, - }); + return new PLUGIN$1( + Object.assign(otherOptions, { + main: Process, + }), + ); } if (Process instanceof Object) { return new PLUGIN$1(Process); @@ -17106,6 +17158,7 @@ const Download = function (options = {}) { * SPDX-License-Identifier: Apache-2.0 */ const loaderFunction = { + // 加载 js 文件到 html 文档中 script(url) { return new Promise((resolve, reject) => { const script = document.createElement('script'); @@ -17120,6 +17173,7 @@ const loaderFunction = { document.body.append(script); }); }, + // 加载 css 文件到 html 文档中 css(url) { return new Promise((resolve, reject) => { const style = document.createElement('style'); @@ -17160,6 +17214,10 @@ var scriptMap = { 'https://unpkg.com/react@16/umd/react.production.min.js', 'https://unpkg.com/react-dom@16/umd/react-dom.production.min.js', ], + vue: [ + 'https://cdn.jsdelivr.net/npm/vue@2.6.14/dist/vue.min.js', + 'https://cdn.jsdelivr.net/npm/vuex@3.6.2/dist/vuex.min.js', + ], }; /** @@ -18081,7 +18139,7 @@ class PLUGIN { name, // 名称,一般用作提示标记 main, // Plugin 中的功能性函数 init, // 初始化整个 Plugin 的函数 - error, // 函数错误时的事件 + error, // operator 错误时的事件,若返回 false 类的数据,将会中断流,返回正常数据会继续流 complete, // 函数完成时的提示事件 options, // main 函数接收的 options saveResult, // 是否保存结果到每一个 Task 中 @@ -18096,24 +18154,18 @@ class PLUGIN { return of(task).pipe( // 设置跳过 Plugin 的逻辑 switchMap((task) => { - if (task.$checkRepeat(this.uuid) || this.forceRetry) { - return of(task).pipe( - map$1((task) => [task.$commit('start', this.uuid), task._originData]), - - switchMap(([data, originData]) => { - const result = this.main(data, originData); - return result instanceof Promise || result instanceof Observable - ? from(result) - : of(result); - }), - map$1((result) => { - task.$commit('success', result, this.uuid, this.saveResult); - return task; - }), - ); - } - console.log('跳过一个目标'); - return of(task); + return of(task).pipe( + map$1((task) => [task.$commit('start', this.uuid), task._originData]), + + switchMap(([data, originData]) => { + const result = this.main(data, originData); + return result instanceof Promise || result instanceof Observable ? from(result) : of(result); + }), + map$1((result) => { + task.$commit('success', result, this.uuid, this.saveResult); + return task; + }), + ); }), // 捕获到异常 catchError((...args) => { @@ -18137,11 +18189,13 @@ class PLUGIN { } } -function Plugin(Process) { +function Plugin(Process, otherOptions = {}) { if (Process instanceof Function) { - return new PLUGIN({ - main: Process, - }); + return new PLUGIN( + Object.assign(otherOptions, { + main: Process, + }), + ); } if (Process instanceof Object) { return new PLUGIN(Process); @@ -18160,7 +18214,7 @@ var index = Object.assign(Spider, tools, { Task, TaskGroup: TaskGroup$1, version: "3.1.8", - buildDate: new Date(1628082806861), + buildDate: new Date(1628151252818), }); export default index; diff --git a/dist/JSpider.min.js b/dist/JSpider.min.js index cad7f14..16ec89c 100644 --- a/dist/JSpider.min.js +++ b/dist/JSpider.min.js @@ -13567,6 +13567,9 @@ var JSpider = (function () { }, start() { + if (self.status === 'pending') throw new Error('Task 处在 pending 状态'); + self.dataSlide = []; + self.dataSlideUUID = ''; self.status = 'pending'; return self.output || self.originData; }, @@ -16432,9 +16435,7 @@ var JSpider = (function () { $isSameTask(task) { return task.$store.spiderUUID === this.$store.spiderUUID && task.$store.uuid === this.$store.uuid; } - $checkRepeat(uuid) { - return this.$store.dataSlide.includes(uuid); - } + $destroy() { this._belongTo = null; this.$commit('destroy'); // 先通知外部,该 Task 被销毁 @@ -16493,17 +16494,60 @@ var JSpider = (function () { * SPDX-License-Identifier: Apache-2.0 */ + /** + * Mirror 是 JSpider 中的数据外放接口 + * + * MessageHub 是集合所有事件的事件中心, + * 当 MessageHub 被 emit 时, MessageHub 发出相关的 Update 进行视图的更新 + * 所有的外放接口使用 rxjs 进行 subscribe + */ + + const MessageHub = new EventHub(); + + // 任何一个 Task 数据发生改变时 + const TaskUpdate = MessageHub.createSource$('TaskUpdate'); + + // ControlPanel 的状态发生改变 + /** + * ControlUpdate + * + * MessageHub.emit('ControlUpdate', payload); + * payload: { + * name:'your message token',// stateChange flowStart flowStop + * value:any, + * } + * + */ + + const ControlUpdate = MessageHub.createSource$('ControlUpdate'); + + /** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ var staticEvent = { // ! 即使这些函数不会被使用,也必须使用注释的方式写下 stateChange(state) { - this.state = state; + this.status = state; + MessageHub.emit('ControlUpdate', { + name: 'stateChange', + value: state, + }); }, 'Flow:stop'() { this._stop = true; + MessageHub.emit('ControlUpdate', { + name: 'flowStop', + }); }, 'Flow:start'() { - console.log('start'); + console.log('jspider 开始运行'); + this.status = 'pending'; this._stop = false; + MessageHub.emit('ControlUpdate', { + name: 'flowStart', + }); }, // 'Flow:input'() {} @@ -16513,11 +16557,16 @@ var JSpider = (function () { if (task instanceof TaskGroup$1) { task.$destroy(); } + MessageHub.emit('ControlUpdate', { + name: 'TaskSuccess', + }); }, 'Task:error'(error) { console.log(error); }, - 'Task:complete'() {}, + // 'Flow:complete'() { + // console.log('线路完成'); + // }, }; /** @@ -16525,26 +16574,11 @@ var JSpider = (function () { * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ + function bindUpdate() { + const backup = this.$store.$backup(); - /** - * Mirror 是 JSpider 中的数据外放接口 - * - * MessageHub 是集合所有事件的事件中心, - * 当 MessageHub 被 emit 时, MessageHub 发出相关的 Update 进行视图的更新 - * 所有的外放接口使用 rxjs 进行 subscribe - */ - - const MessageHub = new EventHub(); - - // Task 数据发生改变时 - const TaskUpdate = MessageHub.createSource$('TaskUpdate'); - - /** - * @license - * Copyright 2021 KonghaYao 江夏尧 - * SPDX-License-Identifier: Apache-2.0 - */ - + MessageHub.emit('TaskUpdate', backup); + } // ! 用于维护全局 Task 数据的中心 class TaskManager { #Tasks = new Map(); // 用于维护所有的 Task @@ -16557,18 +16591,16 @@ var JSpider = (function () { this.#Tasks.set(task.uuid, task); const that = this; task.$on({ + start: bindUpdate, + success: bindUpdate, + complete: bindUpdate, + error: bindUpdate, // 监听事件,并更新响应的 viewModel destroy() { that.#Tasks.delete(this.uuid); // this 绑定的是 task }, }); - ['start', 'success', 'complete', 'error'].forEach((name) => { - task.$on(name, function () { - const backup = this.$store.$backup(); - MessageHub.emit('TaskUpdate', backup); - }); - }); this.viewModel.push(task.$store); return task; } @@ -16628,14 +16660,13 @@ var JSpider = (function () { * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ - // ControlPanel 是 JSpider 内部的事件和数据中心。 // 全部 JSpider 涉及到的边界中,ControlPanel 只有一个,但是 View 可以有多个,而 Spider 就是 View 中的一个 // 用于分发数据流,提供 Task 的状态变更。 // TODO 并且可以提供数据的响应给类似于 UI 界面形成可视化 class ControlPanel$1 { - state = 'free'; // 'free' 'preparing' + status = 'free'; // 'free' 'preparing' #runningQueue = new functionQueue(); // 异步 Queue 队列 _stop = false; // 用于直接切断 spiderSource$ 的流 spiderSource$ = null; @@ -16659,13 +16690,13 @@ var JSpider = (function () { // 所有的事件分配到 staticEvent 中去写 (task) => this.$EventHub.emit('Task:success', task), (error) => this.$EventHub.emit('Task:error', error), - () => this.$EventHub.emit('Task:complete'), + () => this.$EventHub.emit('Flow:complete'), ); this.$EventHub.emit('Flow:stop'); } set pipeline(value) { - if (this.state === 'free') { + if (this.status === 'free') { this.$EventHub.emit('stateChange', 'preparing'); this._pipeline = value; this.#runningQueue.enQueue( @@ -16712,13 +16743,36 @@ var JSpider = (function () { var ControlPanel = new ControlPanel$1(); - /** - * @license - * Copyright 2021 KonghaYao 江夏尧 - * SPDX-License-Identifier: Apache-2.0 - */ + // 注意,这里只要 View 被实例化了,它就会订阅数据 class View { - constructor() {} + constructor({ tasks = false, controlPanel = false } = {}) { + this.config = { + tasks, + controlPanel, + }; + + if (tasks) { + const a = TaskUpdate.subscribe((data) => this._update(data)); + this.#subscriber.push(a); + } + if (controlPanel) { + const b = ControlUpdate.subscribe((data) => this._change(data)); + this.#subscriber.push(b); + } + } + config = {}; + #subscriber = []; + $destroy() { + this.#subscriber.forEach((sub) => { + sub.unsubscribe(); + }); + } + } + + class ConsoleView extends View { + constructor(config) { + super(Object.assign(config, { tasks: true, controlPanel: true })); + } tasks = []; #uuidArray = []; _update(data) { @@ -16731,20 +16785,22 @@ var JSpider = (function () { this.tasks.splice(index, 1, data); } } + _change({ name, value = '' }) {} } - // Spider 是 Console 的数据放送 + + /** + * @license + * Copyright 2021 KonghaYao 江夏尧 + * SPDX-License-Identifier: Apache-2.0 + */ + // Spider 是一个 View class Spider { constructor({ logEvery = false } = {}) { this.config = { logEvery, }; - if (logEvery) { - TaskUpdate.subscribe((data) => { - this.views._update(data); - }); - } + this.views = new ConsoleView(this.config); } - views = new View(); crawl(...args) { ControlPanel.createFlow(args.flat()); return this; @@ -16806,7 +16862,7 @@ var JSpider = (function () { name, // 名称,一般用作提示标记 main, // Plugin 中的功能性函数 init, // 初始化整个 Plugin 的函数 - error, // 函数错误时的事件 + error, // operator 错误时的事件,若返回 false 类的数据,将会中断流,返回正常数据会继续流 complete, // 函数完成时的提示事件 options, // main 函数接收的 options saveResult, // 是否保存结果到每一个 Task 中 @@ -16821,24 +16877,18 @@ var JSpider = (function () { return of(task).pipe( // 设置跳过 Plugin 的逻辑 switchMap((task) => { - if (task.$checkRepeat(this.uuid) || this.forceRetry) { - return of(task).pipe( - map$1((task) => [task.$commit('start', this.uuid), task._originData]), - - switchMap(([data, originData]) => { - const result = this.main(data, originData); - return result instanceof Promise || result instanceof Observable - ? from(result) - : of(result); - }), - map$1((result) => { - task.$commit('success', result, this.uuid, this.saveResult); - return task; - }), - ); - } - console.log('跳过一个目标'); - return of(task); + return of(task).pipe( + map$1((task) => [task.$commit('start', this.uuid), task._originData]), + + switchMap(([data, originData]) => { + const result = this.main(data, originData); + return result instanceof Promise || result instanceof Observable ? from(result) : of(result); + }), + map$1((result) => { + task.$commit('success', result, this.uuid, this.saveResult); + return task; + }), + ); }), // 捕获到异常 catchError((...args) => { @@ -16862,11 +16912,13 @@ var JSpider = (function () { } } - function Plugin$1(Process) { + function Plugin$1(Process, otherOptions = {}) { if (Process instanceof Function) { - return new PLUGIN$1({ - main: Process, - }); + return new PLUGIN$1( + Object.assign(otherOptions, { + main: Process, + }), + ); } if (Process instanceof Object) { return new PLUGIN$1(Process); @@ -17109,6 +17161,7 @@ var JSpider = (function () { * SPDX-License-Identifier: Apache-2.0 */ const loaderFunction = { + // 加载 js 文件到 html 文档中 script(url) { return new Promise((resolve, reject) => { const script = document.createElement('script'); @@ -17123,6 +17176,7 @@ var JSpider = (function () { document.body.append(script); }); }, + // 加载 css 文件到 html 文档中 css(url) { return new Promise((resolve, reject) => { const style = document.createElement('style'); @@ -17163,6 +17217,10 @@ var JSpider = (function () { 'https://unpkg.com/react@16/umd/react.production.min.js', 'https://unpkg.com/react-dom@16/umd/react-dom.production.min.js', ], + vue: [ + 'https://cdn.jsdelivr.net/npm/vue@2.6.14/dist/vue.min.js', + 'https://cdn.jsdelivr.net/npm/vuex@3.6.2/dist/vuex.min.js', + ], }; /** @@ -18084,7 +18142,7 @@ var JSpider = (function () { name, // 名称,一般用作提示标记 main, // Plugin 中的功能性函数 init, // 初始化整个 Plugin 的函数 - error, // 函数错误时的事件 + error, // operator 错误时的事件,若返回 false 类的数据,将会中断流,返回正常数据会继续流 complete, // 函数完成时的提示事件 options, // main 函数接收的 options saveResult, // 是否保存结果到每一个 Task 中 @@ -18099,24 +18157,18 @@ var JSpider = (function () { return of(task).pipe( // 设置跳过 Plugin 的逻辑 switchMap((task) => { - if (task.$checkRepeat(this.uuid) || this.forceRetry) { - return of(task).pipe( - map$1((task) => [task.$commit('start', this.uuid), task._originData]), - - switchMap(([data, originData]) => { - const result = this.main(data, originData); - return result instanceof Promise || result instanceof Observable - ? from(result) - : of(result); - }), - map$1((result) => { - task.$commit('success', result, this.uuid, this.saveResult); - return task; - }), - ); - } - console.log('跳过一个目标'); - return of(task); + return of(task).pipe( + map$1((task) => [task.$commit('start', this.uuid), task._originData]), + + switchMap(([data, originData]) => { + const result = this.main(data, originData); + return result instanceof Promise || result instanceof Observable ? from(result) : of(result); + }), + map$1((result) => { + task.$commit('success', result, this.uuid, this.saveResult); + return task; + }), + ); }), // 捕获到异常 catchError((...args) => { @@ -18140,11 +18192,13 @@ var JSpider = (function () { } } - function Plugin(Process) { + function Plugin(Process, otherOptions = {}) { if (Process instanceof Function) { - return new PLUGIN({ - main: Process, - }); + return new PLUGIN( + Object.assign(otherOptions, { + main: Process, + }), + ); } if (Process instanceof Object) { return new PLUGIN(Process); @@ -18163,7 +18217,7 @@ var JSpider = (function () { Task, TaskGroup: TaskGroup$1, version: "3.1.8", - buildDate: new Date(1628082806861), + buildDate: new Date(1628151252818), }); return index; diff --git a/package.json b/package.json index e459806..803a385 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,9 @@ "scripts": { "dev": "rollup --config rollup.config.dev.js -w", "build": "rollup --config rollup.config.build.js", - "lint": "eslint src plugins FakeServer tools", + "lint": "eslint src package", "Lint": "npm run Prettier && npm run lint", - "Prettier": "prettier -w FakeServer/**/*.{ts,json,md,yml,js} plugins/**/*.{ts,json,md,yml,js} src/**/*.{ts,json,md,yml,js} tools/**/*.{ts,json,md,yml,js} docs/**/*.md", + "Prettier": "prettier -w src/**/*.{ts,json,md,yml,js} package/**/*.{ts,json,md,yml,js} docs/**/*.md", "graph": "madge --image docs/assets/dependencies.svg ./index.js", "release": "standard-version", "lint-staged": "lint-staged", @@ -59,7 +59,6 @@ } }, "dependencies": { - "chalk": "^4.1.1", "consola": "^2.15.3", "lodash-es": "^4.17.21", "mobx": "^6.3.2", diff --git a/package/FakeServer/.eslintignore b/package/FakeServer/.eslintignore new file mode 100644 index 0000000..d1346c8 --- /dev/null +++ b/package/FakeServer/.eslintignore @@ -0,0 +1,2 @@ +**/node_modules/* +**/dist/* diff --git a/FakeServer/.prettierrc b/package/FakeServer/.prettierrc similarity index 100% rename from FakeServer/.prettierrc rename to package/FakeServer/.prettierrc diff --git a/FakeServer/FakeServer.drawio b/package/FakeServer/FakeServer.drawio similarity index 100% rename from FakeServer/FakeServer.drawio rename to package/FakeServer/FakeServer.drawio diff --git a/FakeServer/ajax/XMLHttpRequest/constant.js b/package/FakeServer/ajax/XMLHttpRequest/constant.js similarity index 100% rename from FakeServer/ajax/XMLHttpRequest/constant.js rename to package/FakeServer/ajax/XMLHttpRequest/constant.js diff --git a/FakeServer/ajax/XMLHttpRequest/defineGetterAndSetter.js b/package/FakeServer/ajax/XMLHttpRequest/defineGetterAndSetter.js similarity index 100% rename from FakeServer/ajax/XMLHttpRequest/defineGetterAndSetter.js rename to package/FakeServer/ajax/XMLHttpRequest/defineGetterAndSetter.js diff --git a/FakeServer/ajax/XMLHttpRequest/xhr.js b/package/FakeServer/ajax/XMLHttpRequest/xhr.js similarity index 100% rename from FakeServer/ajax/XMLHttpRequest/xhr.js rename to package/FakeServer/ajax/XMLHttpRequest/xhr.js diff --git a/FakeServer/ajax/fetch/fetch.js b/package/FakeServer/ajax/fetch/fetch.js similarity index 100% rename from FakeServer/ajax/fetch/fetch.js rename to package/FakeServer/ajax/fetch/fetch.js diff --git a/FakeServer/ajax/fetch/src/INTERNALS.js b/package/FakeServer/ajax/fetch/src/INTERNALS.js similarity index 100% rename from FakeServer/ajax/fetch/src/INTERNALS.js rename to package/FakeServer/ajax/fetch/src/INTERNALS.js diff --git a/FakeServer/ajax/fetch/src/body.js b/package/FakeServer/ajax/fetch/src/body.js similarity index 100% rename from FakeServer/ajax/fetch/src/body.js rename to package/FakeServer/ajax/fetch/src/body.js diff --git a/FakeServer/ajax/fetch/src/body/consumeBody.js b/package/FakeServer/ajax/fetch/src/body/consumeBody.js similarity index 100% rename from FakeServer/ajax/fetch/src/body/consumeBody.js rename to package/FakeServer/ajax/fetch/src/body/consumeBody.js diff --git a/FakeServer/ajax/fetch/src/body/exactContentType.js b/package/FakeServer/ajax/fetch/src/body/exactContentType.js similarity index 100% rename from FakeServer/ajax/fetch/src/body/exactContentType.js rename to package/FakeServer/ajax/fetch/src/body/exactContentType.js diff --git a/FakeServer/ajax/fetch/src/response.js b/package/FakeServer/ajax/fetch/src/response.js similarity index 100% rename from FakeServer/ajax/fetch/src/response.js rename to package/FakeServer/ajax/fetch/src/response.js diff --git a/FakeServer/ajax/fetch/src/utils/is-redirect.js b/package/FakeServer/ajax/fetch/src/utils/is-redirect.js similarity index 100% rename from FakeServer/ajax/fetch/src/utils/is-redirect.js rename to package/FakeServer/ajax/fetch/src/utils/is-redirect.js diff --git a/FakeServer/ajax/fetch/src/utils/is.js b/package/FakeServer/ajax/fetch/src/utils/is.js similarity index 100% rename from FakeServer/ajax/fetch/src/utils/is.js rename to package/FakeServer/ajax/fetch/src/utils/is.js diff --git a/FakeServer/ajax/index.js b/package/FakeServer/ajax/index.js similarity index 100% rename from FakeServer/ajax/index.js rename to package/FakeServer/ajax/index.js diff --git a/FakeServer/dist/index.js b/package/FakeServer/dist/index.js similarity index 100% rename from FakeServer/dist/index.js rename to package/FakeServer/dist/index.js diff --git a/FakeServer/fakeServer.js b/package/FakeServer/fakeServer.js similarity index 100% rename from FakeServer/fakeServer.js rename to package/FakeServer/fakeServer.js diff --git a/FakeServer/index.html b/package/FakeServer/index.html similarity index 100% rename from FakeServer/index.html rename to package/FakeServer/index.html diff --git a/FakeServer/package.json b/package/FakeServer/package.json similarity index 100% rename from FakeServer/package.json rename to package/FakeServer/package.json diff --git a/FakeServer/pnpm-lock.yaml b/package/FakeServer/pnpm-lock.yaml similarity index 100% rename from FakeServer/pnpm-lock.yaml rename to package/FakeServer/pnpm-lock.yaml diff --git a/FakeServer/rollup.config.dev.js b/package/FakeServer/rollup.config.dev.js similarity index 100% rename from FakeServer/rollup.config.dev.js rename to package/FakeServer/rollup.config.dev.js diff --git a/FakeServer/src/Route.js b/package/FakeServer/src/Route.js similarity index 100% rename from FakeServer/src/Route.js rename to package/FakeServer/src/Route.js diff --git a/FakeServer/src/Server.js b/package/FakeServer/src/Server.js similarity index 100% rename from FakeServer/src/Server.js rename to package/FakeServer/src/Server.js diff --git a/FakeServer/src/parseURL.js b/package/FakeServer/src/parseURL.js similarity index 100% rename from FakeServer/src/parseURL.js rename to package/FakeServer/src/parseURL.js diff --git a/package/jspider-ui/.eslintignore b/package/jspider-ui/.eslintignore new file mode 100644 index 0000000..d1346c8 --- /dev/null +++ b/package/jspider-ui/.eslintignore @@ -0,0 +1,2 @@ +**/node_modules/* +**/dist/* diff --git a/package/jspider-ui/.eslintrc.js b/package/jspider-ui/.eslintrc.js index c7e1d36..bb3ceec 100644 --- a/package/jspider-ui/.eslintrc.js +++ b/package/jspider-ui/.eslintrc.js @@ -23,7 +23,6 @@ module.exports = { indent: 0, // prettier 已经处理好了 'object-curly-spacing': 0, // prettier 自动格式化这个部分 'max-len': ['error', 120], - 'operator-linebreak': ['error', 'before'], 'quote-props': ['error', 'as-needed'], 'operator-linebreak': 0, 'no-unused-vars': 1, diff --git a/package/jspider-ui/package.json b/package/jspider-ui/package.json index 6bdebca..c5f5794 100644 --- a/package/jspider-ui/package.json +++ b/package/jspider-ui/package.json @@ -34,5 +34,9 @@ "rollup-plugin-terser": "^7.0.2", "rollup-plugin-vue": "5.1.9", "vue-template-compiler": "^2.6.14" + }, + "dependencies": { + "vue": "^2.6.14", + "vuex": "^3.6.2" } } diff --git a/package/jspider-ui/pnpm-lock.yaml b/package/jspider-ui/pnpm-lock.yaml index 155af46..4eb80a3 100644 --- a/package/jspider-ui/pnpm-lock.yaml +++ b/package/jspider-ui/pnpm-lock.yaml @@ -18,7 +18,13 @@ specifiers: rollup-plugin-serve: ^1.1.0 rollup-plugin-terser: ^7.0.2 rollup-plugin-vue: 5.1.9 + vue: 2.6.14 vue-template-compiler: ^2.6.14 + vuex: ^3.6.2 + +dependencies: + vue: 2.6.14 + vuex: 3.6.2_vue@2.6.14 devDependencies: '@babel/core': 7.14.6 @@ -4056,6 +4062,18 @@ packages: resolution: {integrity: sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==} dev: true + /vue/2.6.14: + resolution: {integrity: sha512-x2284lgYvjOMj3Za7kqzRcUSxBboHqtgRE2zlos1qWaOye5yUmHn42LB1250NJBLRwEcdrB0JRwyPTEPhfQjiQ==} + dev: false + + /vuex/3.6.2_vue@2.6.14: + resolution: {integrity: sha512-ETW44IqCgBpVomy520DT5jf8n0zoCac+sxWnn+hMe/CzaSejb/eVw2YToiXYX+Ex/AuHHia28vWTq4goAexFbw==} + peerDependencies: + vue: ^2.0.0 + dependencies: + vue: 2.6.14 + dev: false + /which-boxed-primitive/1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} dependencies: diff --git a/package/jspider-ui/rollup.config.build.js b/package/jspider-ui/rollup.config.build.js index 727d3d9..8936b15 100644 --- a/package/jspider-ui/rollup.config.build.js +++ b/package/jspider-ui/rollup.config.build.js @@ -6,37 +6,44 @@ import del from 'rollup-plugin-delete'; import json from '@rollup/plugin-json'; import replace from '@rollup/plugin-replace'; import CONFIG from './package.json'; - +import vue from 'rollup-plugin-vue'; export default [ { input: 'src/index.js', // 通用版本打包 - plugins: [del({ targets: 'dist/*' }), vue({ - // 把单文件组件中的样式,插入到html中的style标签 - css: true, - // 把组件转换成 render 函数 - compileTemplate: true - }), - replace({ - preventAssignment: true, - values: { __version__: JSON.stringify(CONFIG.version), __buildDate__: new Date().getTime() }, - }), - json(), - resolve({ - browser: true, - }), - commonjs(), // 将 CommonJS 转换成 ES2015 模块供 Rollup 处理 - terser(), - license({ - banner: { - content: { - file: './LICENSE', + plugins: [ + del({ targets: 'dist/*' }), + vue({ + // 把单文件组件中的样式,插入到html中的style标签 + css: true, + // 把组件转换成 render 函数 + compileTemplate: true, + }), + replace({ + preventAssignment: true, + values: { + __version__: JSON.stringify(CONFIG.version), + __buildDate__: new Date().getTime(), + 'process.env.NODE_ENV': JSON.stringify('production'), }, - }, - }),], - output:{ + }), + json(), + resolve({ + browser: true, + }), + commonjs(), // 将 CommonJS 转换成 ES2015 模块供 Rollup 处理 + terser(), + license({ + banner: { + content: { + file: './LICENSE', + }, + }, + }), + ], + output: { // 打包出口 file: './dist/jspider-user-interface.js', format: 'es', - } + }, }, ]; diff --git a/package/jspider-ui/rollup.config.dev.js b/package/jspider-ui/rollup.config.dev.js index e57b013..461cc46 100644 --- a/package/jspider-ui/rollup.config.dev.js +++ b/package/jspider-ui/rollup.config.dev.js @@ -4,7 +4,6 @@ import json from '@rollup/plugin-json'; import replace from '@rollup/plugin-replace'; import CONFIG from './package.json'; import livereload from 'rollup-plugin-livereload'; -import serve from 'rollup-plugin-serve'; import vue from 'rollup-plugin-vue'; import postcss from 'rollup-plugin-postcss'; export default { @@ -33,7 +32,11 @@ export default { }), replace({ preventAssignment: true, - values: { __version__: JSON.stringify(CONFIG.version), __buildDate__: new Date().getTime() }, + values: { + __version__: JSON.stringify(CONFIG.version), + __buildDate__: new Date().getTime(), + 'process.env.NODE_ENV': JSON.stringify('production'), + }, }), json(), resolve({ @@ -41,9 +44,5 @@ export default { }), commonjs(), // 将 CommonJS 转换成 ES2015 模块供 Rollup 处理 livereload({ watch: 'dist' }), - serve({ - open: 'true', - port: '8888', - }), ], }; diff --git a/package/jspider-ui/src/App.vue b/package/jspider-ui/src/App.vue index 01cc030..855ddda 100644 --- a/package/jspider-ui/src/App.vue +++ b/package/jspider-ui/src/App.vue @@ -46,13 +46,16 @@ export default { -moz-osx-font-smoothing: grayscale; text-align: center; color: #2c3e50; - + box-sizing: border-box; position: fixed; top: 0; left: 0; height: 100%; width: 100%; } +.JSpider-UI * { + box-sizing: border-box; +} .card { font-family: 'Open Sans', -apple-system, BlinkMacSystemFont, 'Helvetica Neue', Helvetica, Segoe UI, Arial, Roboto, 'PingFang SC', 'miui', 'Hiragino Sans GB', 'Microsoft Yahei', sans-serif; diff --git a/package/jspider-ui/src/store.js b/package/jspider-ui/src/store.js index c0a904d..2343895 100644 --- a/package/jspider-ui/src/store.js +++ b/package/jspider-ui/src/store.js @@ -1,3 +1,4 @@ +import Vuex from 'vuex'; export const store = new Vuex.Store({ state: { panelVisible: true, diff --git a/plugins/Dexie.js b/package/plugins/Dexie.js similarity index 100% rename from plugins/Dexie.js rename to package/plugins/Dexie.js diff --git a/plugins/Dexie/Dexie.js b/package/plugins/Dexie/Dexie.js similarity index 100% rename from plugins/Dexie/Dexie.js rename to package/plugins/Dexie/Dexie.js diff --git a/plugins/Dexie/data.js b/package/plugins/Dexie/data.js similarity index 100% rename from plugins/Dexie/data.js rename to package/plugins/Dexie/data.js diff --git a/plugins/Download.js b/package/plugins/Download.js similarity index 100% rename from plugins/Download.js rename to package/plugins/Download.js diff --git a/plugins/ExcelHelper.js b/package/plugins/ExcelHelper.js similarity index 100% rename from plugins/ExcelHelper.js rename to package/plugins/ExcelHelper.js diff --git a/plugins/ExcelHelper/createExcelFile.js b/package/plugins/ExcelHelper/createExcelFile.js similarity index 100% rename from plugins/ExcelHelper/createExcelFile.js rename to package/plugins/ExcelHelper/createExcelFile.js diff --git a/plugins/ExcelHelper/xlsx.js b/package/plugins/ExcelHelper/xlsx.js similarity index 100% rename from plugins/ExcelHelper/xlsx.js rename to package/plugins/ExcelHelper/xlsx.js diff --git a/plugins/HTMLParser.js b/package/plugins/HTMLParser.js similarity index 100% rename from plugins/HTMLParser.js rename to package/plugins/HTMLParser.js diff --git a/plugins/JSzip/JSzip.js b/package/plugins/JSzip/JSzip.js similarity index 100% rename from plugins/JSzip/JSzip.js rename to package/plugins/JSzip/JSzip.js diff --git a/plugins/JSzip/zipper.js b/package/plugins/JSzip/zipper.js similarity index 100% rename from plugins/JSzip/zipper.js rename to package/plugins/JSzip/zipper.js diff --git a/plugins/LICENSE b/package/plugins/LICENSE similarity index 100% rename from plugins/LICENSE rename to package/plugins/LICENSE diff --git a/plugins/Request.js b/package/plugins/Request.js similarity index 100% rename from plugins/Request.js rename to package/plugins/Request.js diff --git a/plugins/index.js b/package/plugins/index.js similarity index 100% rename from plugins/index.js rename to package/plugins/index.js diff --git a/plugins/package.json b/package/plugins/package.json similarity index 100% rename from plugins/package.json rename to package/plugins/package.json diff --git a/plugins/utils/cartesianProductOf.js b/package/plugins/utils/cartesianProductOf.js similarity index 100% rename from plugins/utils/cartesianProductOf.js rename to package/plugins/utils/cartesianProductOf.js diff --git a/plugins/utils/toFile.js b/package/plugins/utils/toFile.js similarity index 100% rename from plugins/utils/toFile.js rename to package/plugins/utils/toFile.js diff --git a/plugins/zipFile.js b/package/plugins/zipFile.js similarity index 100% rename from plugins/zipFile.js rename to package/plugins/zipFile.js diff --git a/tools/Mock/Mock.js b/package/tools/Mock/Mock.js similarity index 100% rename from tools/Mock/Mock.js rename to package/tools/Mock/Mock.js diff --git a/tools/Mock/Server/excel.js b/package/tools/Mock/Server/excel.js similarity index 100% rename from tools/Mock/Server/excel.js rename to package/tools/Mock/Server/excel.js diff --git a/tools/Mock/Server/index.js b/package/tools/Mock/Server/index.js similarity index 100% rename from tools/Mock/Server/index.js rename to package/tools/Mock/Server/index.js diff --git a/tools/analysis/Observer/hook.js b/package/tools/analysis/Observer/hook.js similarity index 100% rename from tools/analysis/Observer/hook.js rename to package/tools/analysis/Observer/hook.js diff --git a/tools/analysis/Observer/hook/asyncApply.js b/package/tools/analysis/Observer/hook/asyncApply.js similarity index 100% rename from tools/analysis/Observer/hook/asyncApply.js rename to package/tools/analysis/Observer/hook/asyncApply.js diff --git a/tools/analysis/Observer/hook/createProperty.js b/package/tools/analysis/Observer/hook/createProperty.js similarity index 100% rename from tools/analysis/Observer/hook/createProperty.js rename to package/tools/analysis/Observer/hook/createProperty.js diff --git a/tools/analysis/Observer/hook/syncApply.js b/package/tools/analysis/Observer/hook/syncApply.js similarity index 100% rename from tools/analysis/Observer/hook/syncApply.js rename to package/tools/analysis/Observer/hook/syncApply.js diff --git a/tools/analysis/Observer/watch.js b/package/tools/analysis/Observer/watch.js similarity index 100% rename from tools/analysis/Observer/watch.js rename to package/tools/analysis/Observer/watch.js diff --git a/tools/analysis/Search/GlobalVars.js b/package/tools/analysis/Search/GlobalVars.js similarity index 100% rename from tools/analysis/Search/GlobalVars.js rename to package/tools/analysis/Search/GlobalVars.js diff --git a/tools/analysis/Search/Search.js b/package/tools/analysis/Search/Search.js similarity index 100% rename from tools/analysis/Search/Search.js rename to package/tools/analysis/Search/Search.js diff --git a/tools/analysis/Search/searchObj.js b/package/tools/analysis/Search/searchObj.js similarity index 97% rename from tools/analysis/Search/searchObj.js rename to package/tools/analysis/Search/searchObj.js index 17a3daf..594bf2e 100644 --- a/tools/analysis/Search/searchObj.js +++ b/package/tools/analysis/Search/searchObj.js @@ -9,7 +9,7 @@ * @Last Modified by: KonghaYao * @Last Modified time: 2021-06-29 16:16:22 */ -import getType from '../../../src/utils/type.js'; +import getType from '../../../../src/utils/type.js'; const TypeMap = { RE: '', diff --git a/tools/analysis/Search/window-default.json b/package/tools/analysis/Search/window-default.json similarity index 100% rename from tools/analysis/Search/window-default.json rename to package/tools/analysis/Search/window-default.json diff --git a/tools/analysis/antiDebugger.js b/package/tools/analysis/antiDebugger.js similarity index 100% rename from tools/analysis/antiDebugger.js rename to package/tools/analysis/antiDebugger.js diff --git a/tools/analysis/copy.js b/package/tools/analysis/copy.js similarity index 100% rename from tools/analysis/copy.js rename to package/tools/analysis/copy.js diff --git a/tools/analysis/index.js b/package/tools/analysis/index.js similarity index 100% rename from tools/analysis/index.js rename to package/tools/analysis/index.js diff --git a/tools/index.js b/package/tools/index.js similarity index 100% rename from tools/index.js rename to package/tools/index.js diff --git a/tools/loader/jsDelivr.js b/package/tools/loader/jsDelivr.js similarity index 100% rename from tools/loader/jsDelivr.js rename to package/tools/loader/jsDelivr.js diff --git a/tools/loader/loader.js b/package/tools/loader/loader.js similarity index 100% rename from tools/loader/loader.js rename to package/tools/loader/loader.js diff --git a/tools/loader/loaderFunction.js b/package/tools/loader/loaderFunction.js similarity index 92% rename from tools/loader/loaderFunction.js rename to package/tools/loader/loaderFunction.js index 151e6d3..5f803ab 100644 --- a/tools/loader/loaderFunction.js +++ b/package/tools/loader/loaderFunction.js @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ const loaderFunction = { + // 加载 js 文件到 html 文档中 script(url) { return new Promise((resolve, reject) => { const script = document.createElement('script'); @@ -18,6 +19,7 @@ const loaderFunction = { document.body.append(script); }); }, + // 加载 css 文件到 html 文档中 css(url) { return new Promise((resolve, reject) => { const style = document.createElement('style'); diff --git a/tools/loader/scriptStore.js b/package/tools/loader/scriptStore.js similarity index 88% rename from tools/loader/scriptStore.js rename to package/tools/loader/scriptStore.js index 0b3480b..120cba9 100644 --- a/tools/loader/scriptStore.js +++ b/package/tools/loader/scriptStore.js @@ -22,4 +22,8 @@ export default { 'https://unpkg.com/react@16/umd/react.production.min.js', 'https://unpkg.com/react-dom@16/umd/react-dom.production.min.js', ], + vue: [ + 'https://cdn.jsdelivr.net/npm/vue@2.6.14/dist/vue.min.js', + 'https://cdn.jsdelivr.net/npm/vuex@3.6.2/dist/vuex.min.js', + ], }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e16c124..e80500d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,6 @@ specifiers: '@rollup/plugin-multi-entry': ^4.0.0 '@rollup/plugin-node-resolve': ^13.0.0 '@rollup/plugin-replace': ^2.4.2 - chalk: ^4.1.1 consola: ^2.15.3 cz-conventional-changelog: ^3.3.0 eslint: ^7.29.0 @@ -36,7 +35,6 @@ specifiers: uuid: ^8.3.2 dependencies: - chalk: 4.1.1 consola: 2.15.3 lodash-es: 4.17.21 mobx: 6.3.2 @@ -1629,6 +1627,7 @@ packages: engines: {node: '>=8'} dependencies: color-convert: 2.0.1 + dev: true /anymatch/3.1.2: resolution: {integrity: sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==} @@ -1804,6 +1803,7 @@ packages: dependencies: ansi-styles: 4.3.0 supports-color: 7.2.0 + dev: true /chardet/0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} @@ -1859,6 +1859,7 @@ packages: engines: {node: '>=7.0.0'} dependencies: color-name: 1.1.4 + dev: true /color-name/1.1.3: resolution: {integrity: sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=} @@ -1866,6 +1867,7 @@ packages: /color-name/1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: true /colorette/1.2.2: resolution: {integrity: sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==} @@ -2838,6 +2840,7 @@ packages: /has-flag/4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + dev: true /has-symbols/1.0.2: resolution: {integrity: sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==} @@ -4222,6 +4225,7 @@ packages: engines: {node: '>=8'} dependencies: has-flag: 4.0.0 + dev: true /table/6.7.1: resolution: {integrity: sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg==} diff --git a/rollup.config.dev.js b/rollup.config.dev.js index 5a2bfb8..f7fe0e4 100644 --- a/rollup.config.dev.js +++ b/rollup.config.dev.js @@ -25,10 +25,10 @@ export default { plugins: [ alias({ entries: { - '@tools': './tools', + '@tools': './package/tools', '@src': './src', - '@plugins': './plugins', - '@FakeServer': './FakeServer', + '@plugins': './package/plugins', + '@FakeServer': './package/FakeServer', }, }), replace({ diff --git a/src/ControlPanel/ControlPanel.js b/src/ControlPanel/ControlPanel.js index f507800..5e77369 100644 --- a/src/ControlPanel/ControlPanel.js +++ b/src/ControlPanel/ControlPanel.js @@ -8,14 +8,13 @@ import { TaskManager } from '../Mirror/TaskManager'; import { functionQueue } from '../utils/functionQueue'; import { EventHub } from '../utils/EventHub'; import { pauseToggle } from '../utils/pauseToggle'; - // ControlPanel 是 JSpider 内部的事件和数据中心。 // 全部 JSpider 涉及到的边界中,ControlPanel 只有一个,但是 View 可以有多个,而 Spider 就是 View 中的一个 // 用于分发数据流,提供 Task 的状态变更。 // TODO 并且可以提供数据的响应给类似于 UI 界面形成可视化 export class ControlPanel { - state = 'free'; // 'free' 'preparing' + status = 'free'; // 'free' 'preparing' #runningQueue = new functionQueue(); // 异步 Queue 队列 _stop = false; // 用于直接切断 spiderSource$ 的流 spiderSource$ = null; @@ -39,13 +38,13 @@ export class ControlPanel { // 所有的事件分配到 staticEvent 中去写 (task) => this.$EventHub.emit('Task:success', task), (error) => this.$EventHub.emit('Task:error', error), - () => this.$EventHub.emit('Task:complete'), + () => this.$EventHub.emit('Flow:complete'), ); this.$EventHub.emit('Flow:stop'); } set pipeline(value) { - if (this.state === 'free') { + if (this.status === 'free') { this.$EventHub.emit('stateChange', 'preparing'); this._pipeline = value; this.#runningQueue.enQueue( diff --git a/src/ControlPanel/StaticEvent.js b/src/ControlPanel/StaticEvent.js index baf9f66..c08f409 100644 --- a/src/ControlPanel/StaticEvent.js +++ b/src/ControlPanel/StaticEvent.js @@ -4,18 +4,29 @@ * SPDX-License-Identifier: Apache-2.0 */ import { TaskGroup } from '../TaskSystem/TaskGroup'; - +import { MessageHub } from '../Mirror/Mirror'; export default { // ! 即使这些函数不会被使用,也必须使用注释的方式写下 stateChange(state) { - this.state = state; + this.status = state; + MessageHub.emit('ControlUpdate', { + name: 'stateChange', + value: state, + }); }, 'Flow:stop'() { this._stop = true; + MessageHub.emit('ControlUpdate', { + name: 'flowStop', + }); }, 'Flow:start'() { - console.log('start'); + console.log('jspider 开始运行'); + this.status = 'pending'; this._stop = false; + MessageHub.emit('ControlUpdate', { + name: 'flowStart', + }); }, // 'Flow:input'() {} @@ -25,9 +36,14 @@ export default { if (task instanceof TaskGroup) { task.$destroy(); } + MessageHub.emit('ControlUpdate', { + name: 'TaskSuccess', + }); }, 'Task:error'(error) { console.log(error); }, - 'Task:complete'() {}, + // 'Flow:complete'() { + // console.log('线路完成'); + // }, }; diff --git a/src/Mirror/Mirror.js b/src/Mirror/Mirror.js index 1311665..2fbf27b 100644 --- a/src/Mirror/Mirror.js +++ b/src/Mirror/Mirror.js @@ -3,6 +3,7 @@ * Copyright 2021 KonghaYao 江夏尧 * SPDX-License-Identifier: Apache-2.0 */ +import { filter } from 'rxjs/operators'; import { EventHub } from '../utils/EventHub'; /** @@ -15,7 +16,24 @@ import { EventHub } from '../utils/EventHub'; const MessageHub = new EventHub(); -// Task 数据发生改变时 +// 任何一个 Task 数据发生改变时 const TaskUpdate = MessageHub.createSource$('TaskUpdate'); +const watchTasks = function (uuidArray) { + return TaskUpdate.pipe(filter((task) => uuidArray.includes(task.uuid))); +}; -export { MessageHub, TaskUpdate }; +// ControlPanel 的状态发生改变 +/** + * ControlUpdate + * + * MessageHub.emit('ControlUpdate', payload); + * payload: { + * name:'your message token',// stateChange flowStart flowStop + * value:any, + * } + * + */ + +const ControlUpdate = MessageHub.createSource$('ControlUpdate'); + +export { MessageHub, TaskUpdate, ControlUpdate, watchTasks }; diff --git a/src/Mirror/TaskManager.js b/src/Mirror/TaskManager.js index a9c99e7..db35906 100644 --- a/src/Mirror/TaskManager.js +++ b/src/Mirror/TaskManager.js @@ -6,7 +6,11 @@ /* eslint-disable no-invalid-this */ import { Task } from '../TaskSystem/Task'; import { MessageHub } from './Mirror.js'; +function bindUpdate() { + const backup = this.$store.$backup(); + MessageHub.emit('TaskUpdate', backup); +} // ! 用于维护全局 Task 数据的中心 export class TaskManager { #Tasks = new Map(); // 用于维护所有的 Task @@ -19,18 +23,16 @@ export class TaskManager { this.#Tasks.set(task.uuid, task); const that = this; task.$on({ + start: bindUpdate, + success: bindUpdate, + complete: bindUpdate, + error: bindUpdate, // 监听事件,并更新响应的 viewModel destroy() { that.#Tasks.delete(this.uuid); // this 绑定的是 task }, }); - ['start', 'success', 'complete', 'error'].forEach((name) => { - task.$on(name, function () { - const backup = this.$store.$backup(); - MessageHub.emit('TaskUpdate', backup); - }); - }); this.viewModel.push(task.$store); return task; } diff --git a/src/Pipeline/PluginSystem.js b/src/Pipeline/PluginSystem.js index 73a89ef..24a3eb3 100644 --- a/src/Pipeline/PluginSystem.js +++ b/src/Pipeline/PluginSystem.js @@ -48,7 +48,7 @@ class PLUGIN { name, // 名称,一般用作提示标记 main, // Plugin 中的功能性函数 init, // 初始化整个 Plugin 的函数 - error, // 函数错误时的事件 + error, // operator 错误时的事件,若返回 false 类的数据,将会中断流,返回正常数据会继续流 complete, // 函数完成时的提示事件 options, // main 函数接收的 options saveResult, // 是否保存结果到每一个 Task 中 @@ -63,24 +63,18 @@ class PLUGIN { return of(task).pipe( // 设置跳过 Plugin 的逻辑 switchMap((task) => { - if (task.$checkRepeat(this.uuid) || this.forceRetry) { - return of(task).pipe( - map((task) => [task.$commit('start', this.uuid), task._originData]), + return of(task).pipe( + map((task) => [task.$commit('start', this.uuid), task._originData]), - switchMap(([data, originData]) => { - const result = this.main(data, originData); - return result instanceof Promise || result instanceof Observable - ? from(result) - : of(result); - }), - map((result) => { - task.$commit('success', result, this.uuid, this.saveResult); - return task; - }), - ); - } - console.log('跳过一个目标'); - return of(task); + switchMap(([data, originData]) => { + const result = this.main(data, originData); + return result instanceof Promise || result instanceof Observable ? from(result) : of(result); + }), + map((result) => { + task.$commit('success', result, this.uuid, this.saveResult); + return task; + }), + ); }), // 捕获到异常 catchError((...args) => { @@ -104,11 +98,13 @@ class PLUGIN { } } -export function Plugin(Process) { +export function Plugin(Process, otherOptions = {}) { if (Process instanceof Function) { - return new PLUGIN({ - main: Process, - }); + return new PLUGIN( + Object.assign(otherOptions, { + main: Process, + }), + ); } if (Process instanceof Object) { return new PLUGIN(Process); diff --git a/src/Spider/index.js b/src/Spider/index.js index 0e625eb..62c66b4 100644 --- a/src/Spider/index.js +++ b/src/Spider/index.js @@ -13,35 +13,15 @@ import { Pipeline } from '../Pipeline/index'; import ControlPanel from '../ControlPanel/index.js'; -import { TaskUpdate } from '../Mirror/Mirror.js'; -class View { - constructor() {} - tasks = []; - #uuidArray = []; - _update(data) { - const index = this.#uuidArray.indexOf(data.uuid); - if (index === -1) { - this.#uuidArray.push(data.uuid); - this.tasks.push(data); - } else { - // 并不是直接赋值,而是通过数组的 splice 方式进行数组的更新,这样可以方便 Vue 渲染 - this.tasks.splice(index, 1, data); - } - } -} -// Spider 是 Console 的数据放送 +import { ConsoleView } from '../View/ConsoleView.js'; +// Spider 是一个 View export class Spider { constructor({ logEvery = false } = {}) { this.config = { logEvery, }; - if (logEvery) { - TaskUpdate.subscribe((data) => { - this.views._update(data); - }); - } + this.views = new ConsoleView(this.config); } - views = new View(); crawl(...args) { ControlPanel.createFlow(args.flat()); return this; diff --git a/src/TaskSystem/Task.js b/src/TaskSystem/Task.js index 631d44e..f7cd2cb 100644 --- a/src/TaskSystem/Task.js +++ b/src/TaskSystem/Task.js @@ -27,9 +27,7 @@ export class Task { $isSameTask(task) { return task.$store.spiderUUID === this.$store.spiderUUID && task.$store.uuid === this.$store.uuid; } - $checkRepeat(uuid) { - return this.$store.dataSlide.includes(uuid); - } + $destroy() { this._belongTo = null; this.$commit('destroy'); // 先通知外部,该 Task 被销毁 diff --git a/src/TaskSystem/TaskState.js b/src/TaskSystem/TaskState.js index 532b834..15c25fe 100644 --- a/src/TaskSystem/TaskState.js +++ b/src/TaskSystem/TaskState.js @@ -34,6 +34,9 @@ const TaskStore = types }, start() { + if (self.status === 'pending') throw new Error('Task 处在 pending 状态'); + self.dataSlide = []; + self.dataSlideUUID = ''; self.status = 'pending'; return self.output || self.originData; }, diff --git a/src/View/ConsoleView.js b/src/View/ConsoleView.js new file mode 100644 index 0000000..66c9d1d --- /dev/null +++ b/src/View/ConsoleView.js @@ -0,0 +1,19 @@ +import { View } from './View.js'; +export class ConsoleView extends View { + constructor(config) { + super(Object.assign(config, { tasks: true, controlPanel: true })); + } + tasks = []; + #uuidArray = []; + _update(data) { + const index = this.#uuidArray.indexOf(data.uuid); + if (index === -1) { + this.#uuidArray.push(data.uuid); + this.tasks.push(data); + } else { + // 并不是直接赋值,而是通过数组的 splice 方式进行数组的更新,这样可以方便 Vue 渲染 + this.tasks.splice(index, 1, data); + } + } + _change({ name, value = '' }) {} +} diff --git a/src/View/MemoryView.js b/src/View/MemoryView.js new file mode 100644 index 0000000..8d599c8 --- /dev/null +++ b/src/View/MemoryView.js @@ -0,0 +1,8 @@ +import { View } from './View'; + +export class MemoryView extends View { + constructor(config) { + super(Object.assign(config, { tasks: true, controlPanel: false })); + } + _update(data) {} +} diff --git a/src/View/View.js b/src/View/View.js new file mode 100644 index 0000000..e29ce11 --- /dev/null +++ b/src/View/View.js @@ -0,0 +1,27 @@ +import { TaskUpdate, ControlUpdate } from '../Mirror/Mirror.js'; + +// 注意,这里只要 View 被实例化了,它就会订阅数据 +export class View { + constructor({ tasks = false, controlPanel = false } = {}) { + this.config = { + tasks, + controlPanel, + }; + + if (tasks) { + const a = TaskUpdate.subscribe((data) => this._update(data)); + this.#subscriber.push(a); + } + if (controlPanel) { + const b = ControlUpdate.subscribe((data) => this._change(data)); + this.#subscriber.push(b); + } + } + config = {}; + #subscriber = []; + $destroy() { + this.#subscriber.forEach((sub) => { + sub.unsubscribe(); + }); + } +} diff --git a/test.html b/test.html index 736f936..5e26917 100644 --- a/test.html +++ b/test.html @@ -6,7 +6,7 @@ Document - + diff --git a/test/test.js b/test/test.js index 9839d30..dd08ae5 100644 --- a/test/test.js +++ b/test/test.js @@ -1,5 +1,7 @@ // 使用 deno 测试一些 rxjs 相关的东西 import mitt from 'mitt'; +import { AsyncSubject } from 'rxjs'; +import { bindCallback } from 'rxjs'; import { EMPTY, iif, of, from, pipe, fromEventPattern, timer, interval, Observable, noop, Subscription } from 'rxjs'; import { map, @@ -24,14 +26,10 @@ import { exhaustMap, } from 'rxjs/operators'; -function a() {} -const source$ = fromEventPattern( - (handle) => { - a(handle); - }, - () => {}, -); - +function a(callback) { + callback(a); +} +const source$ = bindCallback(a); const startTime = new Date().getTime(); const compare = () => Math.ceil((new Date().getTime() - startTime) / 100); source$.subscribe((val) => console.log(compare(), val));