Skip to content

Commit

Permalink
Migrate graphs to AWS S3
Browse files Browse the repository at this point in the history
  • Loading branch information
@vadyushkins
vadyushkins committed May 8, 2021
1 parent 29c9f84 commit 7979dd0
Show file tree
Hide file tree
Showing 5 changed files with 298 additions and 242 deletions.
7 changes: 7 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# IDEA
.idea

# Data
cfpq_data/data

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
Expand Down Expand Up @@ -50,6 +53,10 @@ coverage.xml
.hypothesis/
.pytest_cache/

# Doctest files
test.txt
test.xml

# Translations
*.mo
*.pot
Expand Down
5 changes: 4 additions & 1 deletion cfpq_data/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
experimental analysis of context-free path querying algorithms
"""

__version__ = "0.0.0"
__version__ = "1.0.0"

import cfpq_data.config
from cfpq_data.config import *

import cfpq_data.graphs
from cfpq_data.graphs import *
Expand Down
217 changes: 11 additions & 206 deletions cfpq_data/config.py
View file
Original file line number Diff line number Diff line change
@@ -1,210 +1,15 @@
from pathlib import Path

__all__ = [
"MAIN_FOLDER",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"BUCKET_NAME",
]

MAIN_FOLDER = Path(__file__).parent

DATASET = {
"MemoryAliases": {
"Apache_httpd_2.2.18_pointsto_graph": {
"version_id": "6CfRViYHGLyYvSiKf8kWhwOQpFUvUhrw",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"PostgreSQL_8.3.9_pointsto_graph": {
"version_id": "dlVensEsC3.Tjn3MZipEgfCXTyFc9Gno",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"arch_afterInline": {
"version_id": "oYn672Yd85LFkdAVsW3ppJDAXcvTWEsq",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"block_afterInline": {
"version_id": "yjSfyx3ND7LiqlTZcyk3WQDGO.bdmTed",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"bzip2": {
"version_id": "_V9sUGA3HzqgF22dH34QjMq5wbO48oVL",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"crypto_afterInline": {
"version_id": "ikRlpHT_E5e8iutdRRwyiadm6Y3LB2DS",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"drivers_afterInline": {
"version_id": "Wwlv0L6vKWvPYttwLNWfGESPN6wPCnP.",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"fs_afterInline": {
"version_id": "MkdtntPJDg2zH5EYLPvgbZe5y.x.FAyv",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"gzip": {
"version_id": "Qq7CIGhL6TZvfsOhT2ou64mR9HoxwZjz",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"init_afterInline": {
"version_id": "Qb48SAVUqCy3PnQ5tPGesC4jXtw93kpy",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"ipc_afterInline": {
"version_id": "Hg3IDlRfPYHdXPBfdSib.N7TwPYl9n1S",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"kernel_afterInline": {
"version_id": "hDJdX1VgrfoEzJzQCIcCfa4aTRsttVHN",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"lib_afterInline": {
"version_id": "eOLlCHTuaqG952PnkV1UBEhc0ZInJ7C7",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"ls": {
"version_id": "joxt5Ofql.T9BGD9DzGamhY4w1Dp3Of2",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"mm_afterInline": {
"version_id": "h3zNK21CY54316eQcay6fgsp5QJCnOgs",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"net_afterInline": {
"version_id": "4RT6k1W58sd6GciF2wlk4kDaCD3RPBJ6",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"pr": {
"version_id": "7JVaerYb4Xr6Pee4qU8qJWVKyUlJ5rVr",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"security_afterInline": {
"version_id": "ptv8rwCAPdHuo3ABJpvnr92.pQPTU.6C",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"sound_afterInline": {
"version_id": "C2b1j6J73_8tx6pZNVLtTDyhq8bAW5Zn",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
},
"wc": {
"version_id": "wvpVN9XAqfMvkAO44YYXC5L4vKP3lJLM",
"file_extension": ".xml",
"archive_extension": ".tar.gz"
}
},
"RDF": {
"atom-primitive": {
"version_id": "d.KnhyNlOpor7YAnu6MCOefbTQmDAWyy",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"biomedical-mesure-primitive": {
"version_id": "gz33IPh2tGDGI27Z3mIdBBdNYKyOg_.Y",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"core": {
"version_id": "Uztj4uXtJMtW92Tpi3bnBQ.GR_qBSmlp",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"eclass_514en": {
"version_id": "JMWYihKJ463hxx_c6AXT4RjDTk0HlXlD",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"enzyme": {
"version_id": "dD7wbg9FtbuxYQLlmVsnEWGSFSdXhegZ",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"foaf": {
"version_id": "Z6I8ecv_QvvEQRM8V4zOVODq1n_UW8AR",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"funding": {
"version_id": "gdYtwOO4UfJYNjkjVffwneIe2NHSlpDK",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"generations": {
"version_id": "WYijOVKMwCxlJhGO1_7KPYbvumMgABTv",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"geospecies": {
"version_id": "lw9qej02UyFuHkcBmQ_7RIXDajisTK1P",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"go-hierarchy": {
"version_id": "8Sx964vDBw8plJh8uVR028MD.Hm8SyfQ",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"go": {
"version_id": "g3htMbp43uGbexRrbBO4mzHGeYVDBkR3",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"pathways": {
"version_id": "ZpMYFCBHEftavyxjFWOFnvoLkj6wblcJ",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"people_pets": {
"version_id": "NIbwB0y9iGpKQ.eXVZ6qLqvTgon3Czps",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"pizza": {
"version_id": "ARc81JDYzyL0MuO_9f74gn6VjfNZh9FF",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"skos": {
"version_id": "sYzQ5wxgKcGOw5MqXPqojJBwJKRwOKcr",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"taxonomy-hierarchy": {
"version_id": "UW4M3aO4mx72w_Yq0g6106wpfjGVU9ci",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"taxonomy": {
"version_id": "i_ZqYKddbnqtHkk2HD.CX1Ip8YMtgFhQ",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
},
"travel": {
"version_id": "k7O7zm3_Mj4ZT3BjOHbrPiKm3Yjj0HOB",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"univ-bench": {
"version_id": "bt6cu6_XOImhe5wgSXeNFJIZv3il7.ry",
"file_extension": ".owl",
"archive_extension": ".tar.gz"
},
"wine": {
"version_id": "e.1EQN4f_Z1lN8idPAaUnpxO68_AMbTR",
"file_extension": ".rdf",
"archive_extension": ".tar.gz"
}
}
}
AWS_ACCESS_KEY_ID = "AKIA326NG25W2XT6TBAZ"
AWS_SECRET_ACCESS_KEY = "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"

BUCKET_NAME = "cfpq-data"
Loading

0 comments on commit 7979dd0

Please sign in to comment.