diff --git a/cfpq_data/utils/__init__.py b/cfpq_data/utils/__init__.py deleted file mode 100644 index aab171d7..00000000 --- a/cfpq_data/utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from cfpq_data.utils.utils import * -from cfpq_data.utils.graphs_downloader import * diff --git a/cfpq_data/utils/graphs_downloader.py b/cfpq_data/utils/graphs_downloader.py deleted file mode 100644 index 6f3b418e..00000000 --- a/cfpq_data/utils/graphs_downloader.py +++ /dev/null @@ -1,46 +0,0 @@ -import requests - -from cfpq_data.utils import add_graph_dir - - -# TODO: rewrite to work with Google API -def download_file_from_google_drive(file_id, destination): - download_file_prefix = "https://docs.google.com/uc?export=download" - - session = requests.Session() - - response = session.get(download_file_prefix, params={"id": file_id}, stream=True) - token = get_confirm_token(response) - - if token: - params = {"id": file_id, "confirm": token} - response = session.get(download_file_prefix, params=params, stream=True) - - save_response_content(response, destination) - - -def get_confirm_token(response): - for key, value in response.cookies.items(): - if key.startswith("download_warning"): - return value - - return None - - -def save_response_content(response, destination): - chunk_size = 32768 - - with open(destination, "wb") as f: - for chunk in response.iter_content(chunk_size): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - - -def download_data(graph_group, graph_name, graph_key): - dst = add_graph_dir(graph_group) - - arch_dst = dst / f"{graph_name}.tar.xz" - - download_file_from_google_drive(graph_key, arch_dst) - - return dst diff --git a/cfpq_data/utils/utils.py b/cfpq_data/utils/utils.py deleted file mode 100644 index eb1ba5c8..00000000 --- a/cfpq_data/utils/utils.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -import shutil -from pathlib import Path -from typing import List - -from cfpq_data.config import MAIN_FOLDER - - -def __unpack_archive_listdir(target_dir: Path, arch: str) -> List[str]: - """ - Returns a list of files from the archive - - :param target_dir: folder where the archive will be unpacked - :type target_dir: Path - :param arch: path to archive - :type arch: str - :return: list of files from the archive - :rtype: List[str] - """ - - tmp = target_dir / "tmp" - os.mkdir(tmp) - shutil.unpack_archive(arch, tmp) - result = os.listdir(tmp) - shutil.rmtree(tmp) - return result - - -def unpack_graph(graph_group: str, graph_name: str) -> str: - """ - Unpacks the graph to the desired folder - - :param graph_group: graph group type - :type graph_group: str - :param graph_name: graph name - :type graph_name: str - :return: path to the unpacked graph - :rtype: str - """ - - to = MAIN_FOLDER / "data" / graph_group / "Graphs" - - arch = to / f"{graph_name}.tar.xz" - - shutil.unpack_archive(arch, to) - - graph = __unpack_archive_listdir(to, arch)[0] - - os.remove(arch) - - return os.path.join(to, graph) - - -def clean_dir(name: str) -> None: - """ - Clears the specified folder - - :param name: folder to clear - :type name: str - :return: None - :rtype: None - """ - - path = MAIN_FOLDER / "data" / name / "Graphs" - if os.path.isdir(path): - shutil.rmtree(path) - os.mkdir(path) - - -def add_graph_dir(name: str) -> Path: - """ - Creates a folder for the specified graph type - - :param name: specified graph type - :type name: str - :return: path to folder for the specified graph type - :rtype: Path - """ - - dst = MAIN_FOLDER / "data" / name / "Graphs" - dst.mkdir(parents=True, exist_ok=True) - return dst