Skip to content

Commit

Permalink
[Fixes #12441] GS layers metadata update (#12442)
Browse files Browse the repository at this point in the history 
* fix GS layers metadata update

* Use raw abstract instead of the html contant

* disable wrong attributions set inside sync_instance_with_geoserver

* fix formatting

* make flake happy
  • Loading branch information
@giohappy
giohappy authored Jul 26, 2024
1 parent fffa2d5 commit 2e4dae2
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 87 deletions.
45 changes: 12 additions & 33 deletions geonode/geoserver/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
from django.conf import settings
from django.utils import timezone
from django.db import transaction
from django.contrib.auth import get_user_model
from django.utils.module_loading import import_string
from django.core.exceptions import ImproperlyConfigured
from django.template.loader import render_to_string
Expand Down Expand Up @@ -1546,7 +1545,6 @@ def get_store(cat, name, workspace=None):


def fetch_gs_resource(instance, values, tries):
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 2)
try:
gs_resource = gs_catalog.get_resource(name=instance.name, store=instance.store, workspace=instance.workspace)
except Exception:
Expand Down Expand Up @@ -1587,9 +1585,6 @@ def fetch_gs_resource(instance, values, tries):
else:
msg = f"There isn't a geoserver resource for this layer: {instance.name}"
logger.debug(msg)
if tries >= _max_tries:
# raise GeoNodeException(msg)
return (values, None)
gs_resource = None
return (values, gs_resource)

Expand Down Expand Up @@ -2032,41 +2027,16 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs):

gs_resource = None
if not _is_remote_instance:
values = None
values = {"title": instance.title, "abstract": instance.raw_abstract}
_tries = 0
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 3)

# If the store in None then it's a new instance from an upload,
# only in this case run the geoserver_upload method
if getattr(instance, "overwrite", False):
base_file, info = instance.get_base_file()

# There is no need to process it if there is no file.
if base_file:
from geonode.geoserver.upload import geoserver_upload

gs_name, workspace, values, gs_resource = geoserver_upload(
instance,
base_file.file.path,
instance.owner,
instance.name,
overwrite=True,
title=instance.title,
abstract=instance.abstract,
charset=instance.charset,
)

values, gs_resource = fetch_gs_resource(instance, values, _tries)
while not gs_resource and _tries < _max_tries:
values, gs_resource = fetch_gs_resource(instance, values, _tries)
_tries += 1
time.sleep(3)

# Get metadata links
metadata_links = []
for link in instance.link_set.metadata():
metadata_links.append((link.mime, link.name, link.url))

if gs_resource:
logger.debug(f"Found geoserver resource for this dataset: {instance.name}")
instance.gs_resource = gs_resource
Expand All @@ -2078,8 +2048,17 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs):
setattr(instance, key, get_dataset_storetype(values[key]))

if updatemetadata:
# Get metadata links
metadata_links = []
for link in instance.link_set.metadata():
metadata_links.append((link.mime, link.name, link.url))
gs_resource.metadata_links = metadata_links
default_poc = instance.get_first_contact_of_role(role="poc")

"""
TODO: Attributions must be set on a Layer, not a Resource.
We should retrive gs_catalog.get_layer(name=instance.alternate), and obtain the resource from Layer.resource
but I'm not sure if at this stage the layer is ready. For the moment I disable this block.
default_poc = instance.get_first_contact_of_role(role="pointOfContact")
# Update Attribution link
if default_poc:
# gsconfig now utilizes an attribution dictionary
Expand All @@ -2096,7 +2075,7 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs):
settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL
)
gs_resource.attribution_link = site_url + profile.get_absolute_url()

"""
try:
if settings.RESOURCE_PUBLISHING:
if instance.is_published != gs_resource.advertised:
Expand Down
120 changes: 66 additions & 54 deletions geonode/geoserver/management/commands/sync_geonode_datasets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,21 @@
from geonode.layers.models import Dataset
from geonode.security.views import _perms_info_json
from geonode.base.utils import remove_duplicate_links
from geonode.geoserver.helpers import (
create_gs_thumbnail,
sync_instance_with_geoserver,
set_attributes_from_geoserver
)
from geonode.geoserver.helpers import create_gs_thumbnail, sync_instance_with_geoserver, set_attributes_from_geoserver


def sync_geonode_datasets(
ignore_errors, filter, username,
removeduplicates,
updatepermissions,
updatethumbnails,
updateattributes,
updatebbox):
layers = Dataset.objects.all().order_by('name')
ignore_errors,
filter,
username,
removeduplicates,
updatepermissions,
updatethumbnails,
updateattributes,
updatebbox,
updatemetadata,
):
layers = Dataset.objects.all().order_by("name")
if filter:
layers = layers.filter(name__icontains=filter)
if username:
Expand All @@ -65,6 +65,9 @@ def sync_geonode_datasets(
if updatebbox:
print("Regenerating BBOX...")
sync_instance_with_geoserver(layer.id, updatemetadata=False, updatebbox=True)
if updatemetadata:
print("Updating metadata...")
sync_instance_with_geoserver(layer.id, updatemetadata=True, updatebbox=False)
if removeduplicates:
# remove duplicates
print("Removing duplicate links...")
Expand All @@ -77,6 +80,7 @@ def sync_geonode_datasets(
pass
else:
import traceback

traceback.print_exc()
print("Stopping process because --ignore-errors was not set and an error was found.")
return
Expand All @@ -86,74 +90,80 @@ def sync_geonode_datasets(


class Command(BaseCommand):
help = 'Update the GeoNode layers: permissions (including GeoFence database), statistics, thumbnails'
help = "Update the GeoNode layers: permissions (including GeoFence database), statistics, thumbnails"

def add_arguments(self, parser):
parser.add_argument(
'-i',
'--ignore-errors',
action='store_true',
dest='ignore_errors',
"-i",
"--ignore-errors",
action="store_true",
dest="ignore_errors",
default=False,
help='Stop after any errors are encountered.'
help="Stop after any errors are encountered.",
)
parser.add_argument(
'-d',
'--remove-duplicates',
action='store_true',
dest='removeduplicates',
"-d",
"--remove-duplicates",
action="store_true",
dest="removeduplicates",
default=False,
help='Remove duplicates first.'
help="Remove duplicates first.",
)
parser.add_argument(
'-f',
'--filter',
"-f",
"--filter",
dest="filter",
default=None,
help="Only update data the layers that match the given filter."),
help="Only update data the layers that match the given filter.",
),
parser.add_argument(
'-u',
'--username',
dest="username",
default=None,
help="Only update data owned by the specified username.")
"-u", "--username", dest="username", default=None, help="Only update data owned by the specified username."
)
parser.add_argument(
'--updatepermissions',
action='store_true',
"--updatepermissions",
action="store_true",
dest="updatepermissions",
default=False,
help="Update the layer permissions.")
help="Update the layer permissions.",
)
parser.add_argument(
'--updatethumbnails',
action='store_true',
"--updatethumbnails",
action="store_true",
dest="updatethumbnails",
default=False,
help="Update the layer styles and thumbnails.")
help="Update the layer styles and thumbnails.",
)
parser.add_argument(
'--updateattributes',
action='store_true',
"--updateattributes",
action="store_true",
dest="updateattributes",
default=False,
help="Update the layer attributes.")
help="Update the layer attributes.",
)
parser.add_argument(
"--updatebbox", action="store_true", dest="updatebbox", default=False, help="Update the layer BBOX."
)
parser.add_argument(
'--updatebbox',
action='store_true',
dest="updatebbox",
"--updatemetadata",
action="store_true",
dest="updatemetadata",
default=False,
help="Update the layer BBOX.")
help="Update the Geoserver ayer metadata.",
)

def handle(self, **options):
ignore_errors = options.get('ignore_errors')
removeduplicates = options.get('removeduplicates')
updatepermissions = options.get('updatepermissions')
updatethumbnails = options.get('updatethumbnails')
updateattributes = options.get('updateattributes')
updatebbox = options.get('updatebbox')
filter = options.get('filter')
if not options.get('username'):
ignore_errors = options.get("ignore_errors")
removeduplicates = options.get("removeduplicates")
updatepermissions = options.get("updatepermissions")
updatethumbnails = options.get("updatethumbnails")
updateattributes = options.get("updateattributes")
updatebbox = options.get("updatebbox")
updatemetadata = options.get("updatemetadata")
filter = options.get("filter")
if not options.get("username"):
username = None
else:
username = options.get('username')
username = options.get("username")
sync_geonode_datasets(
ignore_errors,
filter,
Expand All @@ -162,4 +172,6 @@ def handle(self, **options):
updatepermissions,
updatethumbnails,
updateattributes,
updatebbox)
updatebbox,
updatemetadata,
)
9 changes: 9 additions & 0 deletions geonode/layers/api/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from geonode.layers.models import Dataset
from geonode.maps.api.serializers import SimpleMapLayerSerializer, SimpleMapSerializer
from geonode.resource.utils import update_resource
from geonode.resource.manager import resource_manager
from rest_framework.exceptions import NotFound

from geonode.storage.manager import StorageManager
Expand Down Expand Up @@ -81,6 +82,14 @@ def get_serializer_class(self):
return DatasetListSerializer
return DatasetSerializer

def partial_update(self, request, *args, **kwargs):
result = super().partial_update(request, *args, **kwargs)

dataset = self.get_object()
resource_manager.update(dataset.uuid, instance=dataset, notify=True),

return result

@extend_schema(
request=DatasetMetadataSerializer,
methods=["put"],
Expand Down

0 comments on commit 2e4dae2

Please sign in to comment.