Skip to content

Commit

Permalink
feat: 添加 Combine 和 Break plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@KonghaYao
KonghaYao committed Aug 7, 2021
1 parent 8a368a5 commit 1b3c976
Show file tree
Hide file tree
Showing 11 changed files with 248 additions and 214 deletions.
154 changes: 73 additions & 81 deletions dist/JSpider.esm.min.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13263,7 +13263,7 @@ function scheduled(input, scheduler) {
}

/** PURE_IMPORTS_START _Observable,_util_subscribeTo,_scheduled_scheduled PURE_IMPORTS_END */
function from(input, scheduler) {
function from$1(input, scheduler) {
if (!scheduler) {
if (input instanceof Observable) {
return input;
Expand Down Expand Up @@ -13328,7 +13328,7 @@ function mergeMap(project, resultSelector, concurrent) {
concurrent = Number.POSITIVE_INFINITY;
}
if (typeof resultSelector === 'function') {
return function (source) { return source.pipe(mergeMap(function (a, i) { return from(project(a, i)).pipe(map(function (b, ii) { return resultSelector(a, b, i, ii); })); }, concurrent)); };
return function (source) { return source.pipe(mergeMap(function (a, i) { return from$1(project(a, i)).pipe(map(function (b, ii) { return resultSelector(a, b, i, ii); })); }, concurrent)); };
}
else if (typeof resultSelector === 'number') {
concurrent = resultSelector;
Expand Down Expand Up @@ -16113,7 +16113,7 @@ function share() {
/** PURE_IMPORTS_START tslib,_map,_observable_from,_innerSubscribe PURE_IMPORTS_END */
function switchMap(project, resultSelector) {
if (typeof resultSelector === 'function') {
return function (source) { return source.pipe(switchMap(function (a, i) { return from(project(a, i)).pipe(map(function (b, ii) { return resultSelector(a, b, i, ii); })); })); };
return function (source) { return source.pipe(switchMap(function (a, i) { return from$1(project(a, i)).pipe(map(function (b, ii) { return resultSelector(a, b, i, ii); })); })); };
}
return function (source) { return source.lift(new SwitchMapOperator(project)); };
}
Expand Down Expand Up @@ -16376,37 +16376,45 @@ class Task {
* Copyright 2021 KonghaYao 江夏尧 <dongzhongzhidong@qq.com>
* SPDX-License-Identifier: Apache-2.0
*/
class TaskGroup$1 extends Task {
class TaskGroup extends Task {
constructor(TaskArray, spiderUUID = '00000') {
const output = new Set(TaskArray);
super(output, TaskArray?.[0]?.spiderUUID || spiderUUID);
super({}, TaskArray?.[0]?.spiderUUID || spiderUUID);
this.member = new Set(TaskArray);
this.#linkTask();
}
#linkTask() {
this.originData.forEach((task) => {
this.member.forEach((task) => {
task._belongTo = this;
task.$on('destroy', () => this.$removeLink(task));
});
}

// 当第一次 start 的时候，返回的是完全不一样的 Task 的信息
// 经过第一个 start 之后，所有 Task 中的数据被统一，这个标识改为 true
consume = false;
// Plugin 的汇报口
$commit(type, ...payload) {
// 扩散事件
// 扩散事件 Set 类型不能 map
const result = [];
this.originData.forEach((task) => result.push(task.$commit(type, ...payload)));
const selfOutput = this.$EventHub.emit(type, ...payload);
this.member.forEach((task) => result.push(task.$commit(type, ...payload)));
this.$EventHub.emit(type, ...payload);

if (type === 'start' && this._output) return selfOutput[0];
if (this.consume) return result[0];
if (type === 'start') this.consume = true;
return result;
}
// 删除所有的 link
$destroy() {
const copy = [...this.originData];
this.$EventHub.emit('destroy'); // 不进行事件的扩散, 只是自身的报销
return copy;
this.$EventHub.emit('destroy'); // 不进行事件的扩散, 只是自身的报销的消息被传开
this.$store.destroy();
const tasks = this.member;
this.member = null;

return tasks;
}
// 单独删除一个连接
$removeLink(task) {
this.originData.delete(task);
this.member.delete(task);
}
get [Symbol.toStringTag]() {
return 'TaskGroup';
Expand Down Expand Up @@ -16479,7 +16487,7 @@ var staticEvent = {
// 'Spider:clearFlow'(){}
'Task:success'(task) {
task.$commit('complete');
if (task instanceof TaskGroup$1) {
if (task instanceof TaskGroup) {
task.$destroy();
}
MessageHub.emit('ControlUpdate', {
Expand Down Expand Up @@ -16629,7 +16637,8 @@ class Pipeline {
let uuidString = '';
// ! 一次遍历实现取出 operator 和 导出 plugin init 函数的 promise 链，并延长 uuidString 用于创建 UUID
const pipeline = this.Plugins.map((plugin, index) => {
uuidString += plugin.main.toString();
// main 属性和 operator 属性必须存在一个来保证能够生成 uuid
uuidString += (plugin?.main || plugin.operator).toString();

if (plugin.init instanceof Function) {
this.#PluginQueue.enQueue(plugin.init);
Expand Down Expand Up @@ -16871,7 +16880,7 @@ class PLUGIN$1 {
});
}
initUUID(index) {
this.uuid = createUUID(this.main.toString() + index);
this.uuid = createUUID((this?.main || this.operator).toString() + index);
}
// 对 main 函数外包一层，直接启动 main 函数的执行，返回一条流
TaskStarter(task) {
Expand All @@ -16883,7 +16892,7 @@ class PLUGIN$1 {

switchMap(([data, originData]) => {
const result = this.main(data, originData);
return result instanceof Promise || result instanceof Observable ? from(result) : of(result);
return result instanceof Promise || result instanceof Observable ? from$1(result) : of(result);
}),
map((result) => {
task.$commit('success', result, this.uuid, this.saveResult);
Expand Down Expand Up @@ -17000,7 +17009,7 @@ function concurrent(
filter((i) => i.length),
// 无论如何每一组都会被推迟的时间量
delayWhen((_, index) => timer(index * delay)),
mergeMap((array) => from(array)),
mergeMap((array) => from$1(array)),
concatMap(asyncSingle),
);
}
Expand Down Expand Up @@ -17143,7 +17152,7 @@ const aDownload = function (file) {
console.log('%c 下载完成', 'color:green');
};

const download = (data, { DownloadFileName: name }, originData) => {
const download = (data, { DownloadFileName: name } = {}, originData) => {
const file = toFile(data, name || (typeof url === 'string' ? originData.url.replace(/[^\/]*?\//g, '') : ''));
DownloadQueue.add(file);
return null;
Expand Down Expand Up @@ -17418,48 +17427,6 @@ async function zipper(fileArray, zipName) {
return content;
}

/**
* @license
* Copyright 2021 KonghaYao 江夏尧 <dongzhongzhidong@qq.com>
* SPDX-License-Identifier: Apache-2.0
*/
class TaskGroup extends Task {
constructor(TaskArray, spiderUUID = '00000') {
const output = new Set(TaskArray);
super(output, TaskArray?.[0]?.spiderUUID || spiderUUID);
this.#linkTask();
}
#linkTask() {
this.originData.forEach((task) => {
task._belongTo = this;
task.$on('destroy', () => this.$removeLink(task));
});
}
// Plugin 的汇报口
$commit(type, ...payload) {
// 扩散事件
const result = [];
this.originData.forEach((task) => result.push(task.$commit(type, ...payload)));
const selfOutput = this.$EventHub.emit(type, ...payload);

if (type === 'start' && this._output) return selfOutput[0];
return result;
}
// 删除所有的 link
$destroy() {
const copy = [...this.originData];
this.$EventHub.emit('destroy'); // 不进行事件的扩散, 只是自身的报销
return copy;
}
// 单独删除一个连接
$removeLink(task) {
this.originData.delete(task);
}
get [Symbol.toStringTag]() {
return 'TaskGroup';
}
}

/**
* @license
* Copyright 2021 KonghaYao 江夏尧 <dongzhongzhidong@qq.com>
Expand All @@ -17471,30 +17438,53 @@ const ZipFile = function (options = {}) {
return Plugin$1({
init,
name: 'zipFile',
main(data) {
main(blob) {
const { zipFileName } = this.options;
const files = data.map((blob) => toFile(blob, zipFileName));
const files = toFile(blob, zipFileName);
return zipper(files, zipFileName);
},
options,
});
};

/**
* @license
* Copyright 2021 KonghaYao 江夏尧 <dongzhongzhidong@qq.com>
* SPDX-License-Identifier: Apache-2.0
*/
function Combine(number, waitTime = 5000, options = {}) {
return Plugin$1({
name: 'Combine', // 这个 name 是负责进行监控的标志符号
options, // 接收所有的参数，提供给所有函数使用
operator() {
// 复写 operator
const { chunk = 3 } = this.options;
return (source) =>
source.pipe(
bufferTime(5000, undefined, chunk),
bufferTime(waitTime, undefined, number),
filter((i) => i.length), // 必须要进行检测是否为空
concatMap((tasks) =>
this.TaskStarter(
new TaskGroup(tasks),

this.uuid,
),
),
concatMap((tasks) => of(new TaskGroup(tasks))),
);
},
});
};
}
function Break() {
return Plugin$1({
name: 'Break', // 这个 name 是负责进行监控的标志符号
options,
operator() {
return (source) =>
source.pipe(
concatMap((taskGroup) => {
if (taskGroup instanceof TaskGroup) {
return from(taskGroup.$destroy());
} else {
return of(taskGroup);
}
}),
);
},
});
}

/**
* @license
Expand All @@ -17507,7 +17497,9 @@ var plugins = /*#__PURE__*/Object.freeze({
Request: Request,
Download: Download,
ExcelHelper: ExcelHelper,
ZipFile: ZipFile
ZipFile: ZipFile,
Combine: Combine,
Break: Break
});

/**
Expand Down Expand Up @@ -18151,7 +18143,7 @@ class PLUGIN {
});
}
initUUID(index) {
this.uuid = createUUID(this.main.toString() + index);
this.uuid = createUUID((this?.main || this.operator).toString() + index);
}
// 对 main 函数外包一层，直接启动 main 函数的执行，返回一条流
TaskStarter(task) {
Expand All @@ -18163,7 +18155,7 @@ class PLUGIN {

switchMap(([data, originData]) => {
const result = this.main(data, originData);
return result instanceof Promise || result instanceof Observable ? from(result) : of(result);
return result instanceof Promise || result instanceof Observable ? from$1(result) : of(result);
}),
map((result) => {
task.$commit('success', result, this.uuid, this.saveResult);
Expand Down Expand Up @@ -18216,9 +18208,9 @@ var index = Object.assign(Spider, tools, {
plugins,
Plugin,
Task,
TaskGroup: TaskGroup$1,
version: "3.2.0",
buildDate: new Date(1628303434439),
TaskGroup,
version: "3.2.1",
buildDate: new Date(1628326859357),
});

export default index;
Loading

0 comments on commit 1b3c976

Please sign in to comment.