diff --git a/common/domain.py b/common/domain.py index cd1a68b4..3e57cea3 100644 --- a/common/domain.py +++ b/common/domain.py @@ -55,6 +55,8 @@ def registered(self): :return: registered domain result """ + if not settings.use_tld_extract: + return self.string result = self.extract() if result: return result.registered_domain diff --git a/config/setting.py b/config/setting.py index 58967016..4f8070d6 100644 --- a/config/setting.py +++ b/config/setting.py @@ -100,3 +100,9 @@ # 搜索模块设置 enable_recursive_search = False # 递归搜索子域 search_recursive_times = 2 # 递归搜索层数 + +# 网络空间测绘引擎设置 +cam_records_maximum_per_domain = 1000 # 对于单个主域名,在测绘引擎中的最多查询多少条记录,防止泛解析和CDN浪费积分,对 fofa, hunter, quake, zoomeye 生效,最低为100 + +# 是否从输入的数据中使用tldextract提取主域名。若设为 False,OneForAll会直接将输入的域名作为主域名,比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com +use_tld_extract = True \ No newline at end of file diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index f2dc2086..86b59b81 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -23,7 +23,7 @@ def search(self): self.page_num = 1 subdomain_encode = f'domain="{self.domain}"'.encode('utf-8') query_data = base64.b64encode(subdomain_encode) - while True: + while 100 * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) @@ -32,7 +32,7 @@ def search(self): 'qbase64': query_data, 'page': self.page_num, 'full': 'true', - 'size': 1000} + 'size': min(1000, settings.cam_records_maximum_per_domain)} resp = self.get(self.addr, query) if not resp: return @@ -42,7 +42,7 @@ def search(self): break self.subdomains.update(subdomains) size = resp_json.get('size') - if size < 1000: + if size < min(1000, settings.cam_records_maximum_per_domain): break self.page_num += 1 diff --git a/modules/search/hunter_api.py b/modules/search/hunter_api.py index cc1d237a..39436512 100644 --- a/modules/search/hunter_api.py +++ b/modules/search/hunter_api.py @@ -22,7 +22,7 @@ def search(self): self.page_num = 1 subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8') query_data = base64.b64encode(subdomain_encode) - while True: + while 100 * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) diff --git a/modules/search/quake_api.py b/modules/search/quake_api.py index 2e42b8d4..a3c554a1 100644 --- a/modules/search/quake_api.py +++ b/modules/search/quake_api.py @@ -20,7 +20,7 @@ def search(self): """ self.per_page_num = 100 self.page_num = 0 - while True: + while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.header.update({'Content-Type': 'application/json'}) diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index 9afff49b..e27bba04 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -19,7 +19,7 @@ def search(self): """ self.per_page_num = 30 self.page_num = 1 - while True: + while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.header.update({'API-KEY': self.key}) diff --git a/requirements.txt b/requirements.txt index 42460b78..8f195ea6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ certifi==2022.06.15 chardet==5.0.0 colorama==0.4.4 dnspython==2.2.1 -exrex==0.10.5 +exrex==0.11.0 fire==0.4.0 future==0.18.2 idna==3.3 @@ -20,3 +20,4 @@ tqdm==4.64.0 treelib==1.6.1 urllib3==1.26.9 win32-setctime==1.1.0 +setuptools