Skip to content

Commit

Permalink
fix(task group): 解决历史遗留的 _ 前缀的问题
Browse files Browse the repository at this point in the history
  • Loading branch information
KonghaYao committed Aug 7, 2021
1 parent 9b08e07 commit 7136cd3
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 30 deletions.
4 changes: 4 additions & 0 deletions VERSION.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
- 这个使用了 Mobx 进行数据监听,rxjs 进行数据传播,3.3 将会以 Vue 作为可视化的 View 层。

- 重构所有的核心代码

- [x] Control Panel 暂停功能
- [x] Control Panel 重试功能(非流内自动重试)

- Console 的错误提示
-

### 因为是 3.2 版本才更新这个文件,所以往前的版本都丢失了。
12 changes: 6 additions & 6 deletions src/TaskSystem/TaskGroup.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
*/
import { Task } from './Task';
export class TaskGroup extends Task {
constructor(TaskArray, _spiderUUID = '00000') {
constructor(TaskArray, spiderUUID = '00000') {
const output = new Set(TaskArray);
super(output, TaskArray?.[0]?._spiderUUID || _spiderUUID);
super(output, TaskArray?.[0]?.spiderUUID || spiderUUID);
this.#linkTask();
}
#linkTask() {
this._originData.forEach((task) => {
this.originData.forEach((task) => {
task._belongTo = this;
task.$on('destroy', () => this.$removeLink(task));
});
Expand All @@ -20,21 +20,21 @@ export class TaskGroup extends Task {
$commit(type, ...payload) {
// 扩散事件
const result = [];
this._originData.forEach((task) => result.push(task.$commit(type, ...payload)));
this.originData.forEach((task) => result.push(task.$commit(type, ...payload)));
const selfOutput = this.$EventHub.emit(type, ...payload);

if (type === 'start' && this._output) return selfOutput[0];
return result;
}
// 删除所有的 link
$destroy() {
const copy = [...this._originData];
const copy = [...this.originData];
this.$EventHub.emit('destroy'); // 不进行事件的扩散, 只是自身的报销
return copy;
}
// 单独删除一个连接
$removeLink(task) {
this._originData.delete(task);
this.originData.delete(task);
}
get [Symbol.toStringTag]() {
return 'TaskGroup';
Expand Down
57 changes: 33 additions & 24 deletions test/text-bilibili.js
Original file line number Diff line number Diff line change
@@ -1,25 +1,34 @@
import JSpider from 'https://cdn.jsdelivr.net/gh/konghayao/jspider/dist/JSpider.esm.min.js';
let {
plugins,
plugins: { ExcelHelper, Request, Download },
} = JSpider;
let keyword = '';
let first = await fetch(
`https://api.bilibili.com/x/web-interface/search/type?context=&page=1&order=&keyword=${keyword}&duration=&tids_1=&tids_2=&platform=pc&search_type=video&highlight=1&single_column=0`,
{
referrer: 'https://search.bilibili.com/all?keyword=%E5%A5%A5%E8%BF%90',
method: 'GET',
},
).then((res) => res.json());
let urls = [...Array(first.data.numPages).keys()].map((i) => {
return `https://api.bilibili.com/x/web-interface/search/type?context=&page=${i}&order=&keyword=${keyword}&duration=&tids_1=&tids_2=&platform=pc&search_type=video&highlight=1&single_column=0`;
});
import('https://cdn.jsdelivr.net/npm/js-spider@3.2/dist/JSpider.esm.min.js').then(async (res) => {
let JSpider = res.default;
let {
Plugin,
plugins: { ExcelHelper, Request, Download },
} = JSpider;
let keyword = '奥运';
let first = await fetch(
`https://api.bilibili.com/x/web-interface/search/type?context=&page=1&order=&keyword=${keyword}&duration=&tids_1=&tids_2=&from_source=&from_spmid=333.336&platform=pc&__refresh__=true&_extra=&search_type=video&highlight=1&single_column=0`,
{
referrer: 'https://search.bilibili.com/all?keyword=%E5%A5%A5%E8%BF%90',
method: 'GET',
},
).then((res) => res.json());
let urls = [...Array(first.data.numPages).keys()].map((i) => {
return {
url: `https://api.bilibili.com/x/web-interface/search/type?context=&page=${
i + 1
}&order=&keyword=${keyword}&duration=&tids_1=&tids_2=&from_source=&from_spmid=333.336&platform=pc&__refresh__=true&_extra=&search_type=video&highlight=1&single_column=0`,
options: { referrer: 'https://search.bilibili.com/all?keyword=%E5%A5%A5%E8%BF%90', method: 'GET' },
};
});

let spider = new JSpider()
.pipeline(
Request(),
ExcelHelper((data) => data.data.result),
Download(),
)
.crawl(urls)
.start();
let spider = new JSpider()
.pipeline(
Request(),
ExcelHelper((data) => {
return data.data.result;
}),
Download(),
)
.crawl(urls)
.start();
});

0 comments on commit 7136cd3

Please sign in to comment.