Merge pull request #55 from pblottiere/mapproxy_s3_cache

Add MapProxy s3 cache support
pblottiere · Jul 16, 2024 · baa8cfb · baa8cfb
2 parents 0764511 + 1937302
commit baa8cfb
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 16 deletions.
diff --git a/docs/src/qsa-api/configuration.md b/docs/src/qsa-api/configuration.md
@@ -2,14 +2,16 @@
 
 QSA web server can be configured thanks to the next environment variables:
 
-| Mandatory  | Environment variable                   |         Description                                                          |
-|------------|----------------------------------------|------------------------------------------------------------------------------|
-| Yes        | `QSA_QGISSERVER_URL`                   | QGIS Server URL                                                              |
-| Yes        | `QSA_QGISSERVER_PROJECTS_DIR`          | Storage location on the filesystem for QGIS projects/styles and QSA database |
-| No         | `QSA_LOGLEVEL`                         | Loglevel : DEBUG, INFO (default) or ERROR                                    |
-| No         | `QSA_QGISSERVER_PROJECTS_PSQL_SERVICE` | PostgreSQL service to store QGIS projects                                    |
-| No         | `QSA_QGISSERVER_MONITORING_PORT`       | Connection port for `qsa-plugin`                                             |
-| No         | `QSA_MAPPROXY_PROJECTS_DIR`            | Storage location on the filesystem for MapProxy configuration files          |
+| Mandatory  | Environment variable                   |         Description                                                              |
+|------------|----------------------------------------|----------------------------------------------------------------------------------|
+| Yes        | `QSA_QGISSERVER_URL`                   | QGIS Server URL                                                                  |
+| Yes        | `QSA_QGISSERVER_PROJECTS_DIR`          | Storage location on the filesystem for QGIS projects/styles and QSA database     |
+| No         | `QSA_LOGLEVEL`                         | Loglevel : DEBUG, INFO (default) or ERROR                                        |
+| No         | `QSA_QGISSERVER_PROJECTS_PSQL_SERVICE` | PostgreSQL service to store QGIS projects                                        |
+| No         | `QSA_QGISSERVER_MONITORING_PORT`       | Connection port for `qsa-plugin`                                                 |
+| No         | `QSA_MAPPROXY_PROJECTS_DIR`            | Storage location on the filesystem for MapProxy configuration files              |
+| No         | `QSA_MAPPROXY_CACHE_S3_BUCKET`         | Activate S3 cache for MapProxy if bucket is set                                  |
+| No         | `QSA_MAPPROXY_CACHE_S3_DIR`            | S3 cache directory for MapProxy. Default to `/mapproxy/cache`                    |
 
 
 ## PostgreSQL support {#postgresql-support}

diff --git a/qsa-api/qsa_api/config.py b/qsa-api/qsa_api/config.py
@@ -46,3 +46,19 @@ def qgisserver_projects_psql_service(self) -> str:
     @property
     def mapproxy_projects_dir(self) -> str:
         return os.environ.get("QSA_MAPPROXY_PROJECTS_DIR", "").replace('"', "")
+
+    @property
+    def mapproxy_cache_s3_bucket(self) -> str:
+        return os.environ.get("QSA_MAPPROXY_CACHE_S3_BUCKET", "")
+
+    @property
+    def mapproxy_cache_s3_dir(self) -> str:
+        return os.environ.get("QSA_MAPPROXY_CACHE_S3_DIR", "/mapproxy/cache")
+
+    @property
+    def aws_access_key_id(self) -> str:
+        return os.environ.get("AWS_ACCESS_KEY_ID", "")
+
+    @property
+    def aws_secret_access_key(self) -> str:
+        return os.environ.get("AWS_SECRET_ACCESS_KEY", "")
diff --git a/qsa-api/qsa_api/mapproxy/mapproxy.py b/qsa-api/qsa_api/mapproxy/mapproxy.py
@@ -1,12 +1,14 @@
 # coding: utf8
 
+import sys
 import yaml
+import boto3
 import shutil
 from pathlib import Path
 
 from qgis.PyQt.QtCore import Qt, QDateTime
 
-from ..utils import config, qgisserver_base_url
+from ..utils import config, logger, qgisserver_base_url
 
 
 class QSAMapProxy:
@@ -47,9 +49,24 @@ def read(self) -> bool:
         return True, ""
 
     def clear_cache(self, layer_name: str) -> None:
-        cache_dir = self._mapproxy_project.parent / "cache_data"
-        for d in cache_dir.glob(f"{layer_name}_cache_*"):
-            shutil.rmtree(d)
+        if config().mapproxy_cache_s3_bucket:
+            bucket_name = config().mapproxy_cache_s3_bucket
+            cache_dir = f"{config().mapproxy_cache_s3_dir}/{layer_name}"
+            if cache_dir[0] == "/":
+                cache_dir = cache_dir[1:]
+            self.debug(f"Clear S3 cache 's3://{bucket_name}/{cache_dir}'")
+            s3 = boto3.resource(
+                "s3",
+                aws_access_key_id=config().aws_access_key_id,
+                aws_secret_access_key=config().aws_secret_access_key,
+            )
+            bucket = s3.Bucket(bucket_name)
+            bucket.objects.filter(Prefix=cache_dir).delete()
+        else:
+            cache_dir = self._mapproxy_project.parent / "cache_data"
+            self.debug(f"Clear cache '{cache_dir}'")
+            for d in cache_dir.glob(f"{layer_name}_cache_*"):
+                shutil.rmtree(d)
 
     def add_layer(
         self,
@@ -81,6 +98,14 @@ def add_layer(
             c["use_direct_from_level"] = 14
             c["meta_size"] = [1, 1]
             c["meta_buffer"] = 0
+
+        if config().mapproxy_cache_s3_bucket:
+            s3_cache_dir = f"{config().mapproxy_cache_s3_dir}/{name}"
+            c["cache"] = {}
+            c["cache"]["type"] = "s3"
+            c["cache"]["directory"] = s3_cache_dir
+            c["cache"]["bucket_name"] = config().mapproxy_cache_s3_bucket
+
         self.cfg["caches"][f"{name}_cache"] = c
 
         s = {
@@ -107,6 +132,9 @@ def remove_layer(self, name: str) -> None:
         if "layers" not in self.cfg:
             return
 
+        # clear cache
+        self.clear_cache(name)
+
         # clean layers
         layers = []
         for layer in self.cfg["layers"]:
@@ -124,6 +152,11 @@ def remove_layer(self, name: str) -> None:
         if source_name in self.cfg["sources"]:
             self.cfg["sources"].pop(source_name)
 
+    def debug(self, msg: str) -> None:
+        caller = f"{self.__class__.__name__}.{sys._getframe().f_back.f_code.co_name}"
+        msg = f"[{caller}][{self.name}] {msg}"
+        logger().debug(msg)
+
     @staticmethod
     def _mapproxy_projects_dir() -> Path:
         return Path(config().mapproxy_projects_dir)

diff --git a/qsa-api/qsa_api/project.py b/qsa-api/qsa_api/project.py
@@ -303,8 +303,14 @@ def create(self, author: str) -> (bool, str):
         return rc, project.error()
 
     def remove(self) -> None:
+        # clear cache and stuff
+        for layer in self.layers:
+            self.remove_layer(layer)
+
+        # remove qsa projects dir
         shutil.rmtree(self._qgis_project_dir)
 
+        # remove remove qgis prohect in db if necessary
         if StorageBackend.type() == StorageBackend.POSTGRESQL:
             storage = (
                 QgsApplication.instance()
@@ -313,9 +319,6 @@ def remove(self) -> None:
             )
             storage.removeProject(self._qgis_project_uri)
 
-        if self._mapproxy_enabled:
-            QSAMapProxy(self.name).remove()
-
     def add_layer(
         self,
         datasource: str,
@@ -639,7 +642,7 @@ def remove_style(self, name: str) -> bool:
 
         return True, ""
 
-    def debug(self, msg) -> None:
+    def debug(self, msg: str) -> None:
         caller = f"{self.__class__.__name__}.{sys._getframe().f_back.f_code.co_name}"
         if StorageBackend.type() == StorageBackend.FILESYSTEM:
             msg = f"[{caller}][{self.name}] {msg}"