#DIGREPO-935: Process Merritt Atom Feed to ingest ETDs

.env

@@ -0,0 +1,2 @@
+MERRITT_USER=tjohnson
+MERRITT_PASSWORD=fakepwd

.travis.yml

@@ -1,6 +1,7 @@
 ---
 sudo: false
 language: ruby
+dist: trusty
 
 cache:
   bundler: true

Gemfile

@@ -9,6 +9,7 @@ end
 gem "puma"
 gem "rails", "~> 5.1.5"
 
+gem "feedjira"
 gem "font-awesome-sass"
 gem "jquery-rails", "~> 4.0"
 gem "sass-rails", "~> 5.0"
@@ -24,6 +25,7 @@ gem "blacklight_range_limit"
 gem "curation_concerns", "~> 1.7.7"
 gem "ezid-client", "~> 1.2"
 gem "hydra-role-management"
+gem "iso639"
 gem "linked_vocabs",
     git: "https://github.com/projecthydra-labs/linked_vocabs.git"
 gem "marc"
@@ -59,6 +61,7 @@ gem "american_date", "~> 1.1.0"
 group :development, :test do
   gem "awesome_print"
   gem "byebug"
+  gem "dotenv"
   gem "factory_bot_rails"
 
   # Used exact gem versions for solr_wrapper and fcrepo_wrapper

Gemfile.lock

@@ -241,6 +241,7 @@ GEM
       activesupport
     diff-lcs (1.3)
     dot-properties (0.1.3)
+    dotenv (2.7.5)
     dropbox_api (0.1.10)
       faraday (~> 0.9, ~> 0.8)
       oauth2 (~> 1.1)
@@ -298,6 +299,9 @@ GEM
       faraday (>= 0.7.4, < 1.0)
     fcrepo_wrapper (0.7.0)
       ruby-progressbar
+    feedjira (3.0.0)
+      loofah (>= 2.2.1)
+      sax-machine (>= 1.0)
     ffi (1.9.23)
     flot-rails (0.0.7)
       jquery-rails
@@ -391,6 +395,7 @@ GEM
       activesupport (<= 6)
     inflecto (0.0.2)
     iso-639 (0.2.8)
+    iso639 (1.3.2)
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
@@ -721,7 +726,7 @@ GEM
       multipart-post
       oauth2
     ruby-progressbar (1.9.0)
-    rubyzip (1.2.1)
+    rubyzip (1.2.3)
     safe_yaml (1.0.4)
     sass (3.5.6)
       sass-listen (~> 4.0.0)
@@ -734,6 +739,7 @@ GEM
       sprockets (>= 2.8, < 4.0)
       sprockets-rails (>= 2.0, < 4.0)
       tilt (>= 1.1, < 3)
+    sax-machine (1.3.2)
     scrub_rb (1.0.1)
     settingslogic (2.0.9)
     signet (0.8.1)
@@ -864,13 +870,16 @@ DEPENDENCIES
   ci_reporter
   curation_concerns (~> 1.7.7)
   database_cleaner
+  dotenv
   ezid-client (~> 1.2)
   factory_bot_rails
   fcrepo_wrapper (= 0.7.0)
+  feedjira
   font-awesome-sass
   highline
   http_logger
   hydra-role-management
+  iso639
   jbuilder (~> 2.0)
   jquery-rails (~> 4.0)
   kaminari_route_prefix
@@ -915,4 +924,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.16.2
+   1.17.1

app/controllers/catalog_controller.rb

@@ -312,6 +312,9 @@ def enforce_show_permissions(_opts = {})
     config.add_show_field solr_name("identifier", :displayable),
                           label: "ARK"
 
+    config.add_show_field solr_name("merritt_id", :displayable),
+                          label: "Merritt ARK"
+
     config.add_show_field solr_name("accession_number", :symbol),
                           label: "Local Identifier"
 

app/indexers/etd_indexer.rb

@@ -42,10 +42,11 @@ def dissertation
         object.dissertation_year.first
     end
 
-    # Derive department by stripping "UC, SB" from the degree grantor field
     def department(solr_doc)
-      Array(solr_doc["degree_grantor_tesim"]).map do |a|
-        a.sub(/^University of California, Santa Barbara\. /, "")
+      # "degree_grantor_label_tesim"
+      # ["University of California, Santa Barbara.Computer Science"]
+      Array(solr_doc["degree_grantor_label_tesim"]).map do |a|
+        a.split(".").last.strip
       end
     end
 

app/models/concerns/metadata.rb

@@ -39,9 +39,6 @@ module Metadata
       index.as :displayable
     end
 
-    # For Merritt ARKs
-    property :merritt_id, predicate: RDF::Vocab::DC11.identifier
-
     property :accession_number,
              predicate: RDF::URI(
                "http://opaquenamespace.org/ns/cco/accessionNumber"

app/models/etd.rb

@@ -33,6 +33,11 @@ class ETD < ActiveFedora::Base
     index.as :displayable
   end
 
+  # For Merritt ARKs
+  property :merritt_id, predicate: ::RDF::Vocab::DC11.identifier do |index|
+    index.as :displayable, :stored_searchable
+  end
+
   include NestedAttributes
 
   has_subresource :proquest

app/models/merritt/feed.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class Merritt::Feed < ActiveRecord::Base
+  self.table_name_prefix = "merritt_"
+
+  HOME = "https://merritt.cdlib.org"
+
+  def self.etd_feed_url(page = 1)
+    HOME + "/object/recent.atom?collection=ark:/13030/m5pv6m0x&page=#{page}"
+  end
+
+  def self.parse(page = 1)
+    url = etd_feed_url(page)
+    uri = URI.parse(url)
+    request = Net::HTTP::Get.new(uri.request_uri)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    response = http.request request
+
+    if response.code.to_i != 200
+      raise StandardError,
+            "Error in Merritt::Feed.parse:"\
+            "#{url} returned #{response.code} #{response.message}"
+    end
+
+    xml = response.body
+    Feedjira.parse(xml)
+  end
+
+  # Feed pages are available as
+  # links in the following order
+  # current, first, last & next
+  def self.current_page
+    parse.links.first.split("&").last.split("=").last.to_i
+  end
+
+  def self.first_page
+    parse.links[1].split("&").last.split("=").last.to_i
+  end
+
+  def self.last_page
+    parse.links[2].split("&").last.split("=").last.to_i
+  end
+
+  def self.last_modified
+    parse.last_modified
+  end
+end

config/application.yml

@@ -29,6 +29,7 @@ defaults: &defaults
   oai_record_prefix: "oai:ucsb"
   oai_identifier_prefix: "https://alexandria.ucsb.edu/lib/"
   uploads_dir: <%= ENV['ADRL_UPLOADS'] || Rails.root.join("tmp", "uploads") %>
+  merritt_etd_dir: <%= ENV['ADRL_MERRITT_ETD_DIR'] || Rails.root.join("tmp", "merrittetds") %>
 
 development:
   <<: *defaults
@@ -44,3 +45,4 @@ production:
   log_dir: /var/log/samvera
   minter_state: /opt/alexandria/shared/minter-state
   uploads_dir: /opt/alexandria/uploads
+  merritt_etd_dir: /opt/alexandria/merrittetds

config/merritt.yml

@@ -0,0 +1,14 @@
+defaults: &defaults
+  merritt_user: <%= ENV.fetch("MERRITT_USER") %>
+  merritt_pwd: <%= ENV.fetch("MERRITT_PASSWORD") %>
+  merritt_etd_dir: <%= ENV['ADRL_MERRITT_ETD_DIR'] || Rails.root.join("tmp", "merrittetds") %>
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults
+
+production:
+  <<: *defaults
+  merritt_etd_dir: /opt/alexandria/merrittetds

db/migrate/20191010183125_create_merritt_feeds.rb

@@ -0,0 +1,9 @@
+class CreateMerrittFeeds < ActiveRecord::Migration[5.1]
+  def change
+    create_table :merritt_feeds do |t|
+      t.integer  :last_parsed_page, index: true, null: false
+      t.datetime :last_modified, index: true
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170308175556) do
+ActiveRecord::Schema.define(version: 20191010183125) do
 
   create_table "bookmarks", force: :cascade do |t|
     t.integer "user_id", null: false
@@ -56,6 +56,15 @@
     t.index ["user_id"], name: "index_curation_concerns_operations_on_user_id"
   end
 
+  create_table "merritt_feeds", force: :cascade do |t|
+    t.integer "last_parsed_page", null: false
+    t.datetime "last_modified"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["last_modified"], name: "index_merritt_feeds_on_last_modified"
+    t.index ["last_parsed_page"], name: "index_merritt_feeds_on_last_parsed_page"
+  end
+
   create_table "searches", force: :cascade do |t|
     t.text "query_params"
     t.integer "user_id"

doc/ingesting.md

@@ -255,7 +255,6 @@ What happens when you run `bin/ingest -f etd /path/to/etds/etdadmin_upload*`, th
     }
     ```
 
-
 2. Next, `bin/ingest` passes the path to the XML file of each ETD to
    {Importer::ETDParser.parse_file}, which parses the XML and queries
    the SRU API, returning a single string of MARC containing the
@@ -276,7 +275,7 @@ What happens when you run `bin/ingest -f etd /path/to/etds/etdadmin_upload*`, th
 5. {ObjectFactoryWriter#put} tidies up the information extracted from
    the MARC and passes it (as `attributes`) and the ETD hash to
    {ObjectFactoryWriter#build_object}, which in turn creates the
-   {Importer::Factory} for ETDs.
+   {Importer::Factory} for ETDs i.e. #{ETDFactory}.
 
 6. {Importer::Factory::ETDFactory} is what saves the ETD into Fedora.
    It inherits from {Importer::Factory::ObjectFactory} and most of its

lib/identifier.rb

@@ -10,4 +10,13 @@ def self.ark_to_noid(ark)
   def self.ark_to_id(ark)
     ark_to_noid(ark)
   end
+
+  def self.merritt_ark_to_noid(ark)
+    return unless (matches = %r{^ark:/\d{5}/(m\w{7,9})$}.match(ark))
+    matches[1]
+  end
+
+  def self.merritt_ark_to_id(ark)
+    merritt_ark_to_noid(ark)
+  end
 end

lib/merritt/import_etd.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "net/https"
+require "uri"
+
+module Merritt::ImportEtd
+  # metadata, dissertation
+  # & supplemental files
+  def self.import(etd)
+    # tmp/merrittetds
+    merritt_etd_dir = Rails.application.config_for(:merritt)["merritt_etd_dir"]
+    Dir.mkdir merritt_etd_dir unless File.directory? merritt_etd_dir
+    # tmp/merrittetds/download_m5bp580k
+    download_path = File.join(merritt_etd_dir, "download_#{ark(etd)}")
+    Dir.mkdir download_path unless File.directory? download_path
+
+    [metadata_url(etd), dissertation_url(etd)].each do |url|
+      # tmp/merrittetds/download_m5bp580k/Soriano_ucsb_0035N_14420_DATA.xml
+      # tmp/merrittetds/download_m5bp580k/Soriano_ucsb_0035N_14420.pdf
+      file_path = File.join(download_path, escape_encodings(url))
+      create_proquest_file(file_path, url)
+    end
+
+    if supp_urls(etd).present?
+      # tmp/merrittetds/download_m5bp580k/supplements
+      supp_path = File.join(download_path, "supplements")
+      Dir.mkdir(supp_path) unless File.directory?(supp_path)
+
+      supp_urls(etd).each do |supp_url|
+        # tmp/merrittetds/download_m5bp580k/supplements/av.jpeg
+        supp_file_path = File.join(supp_path,
+                                   escape_encodings(supp_url))
+        create_proquest_file(supp_file_path, supp_url)
+      end
+    end
+    download_path
+  end
+
+  def self.ark(etd)
+    etd.entry_id.split("/").last
+  end
+
+  # includes proquest pdf, xml
+  # & optional supplemental files
+  def self.proquest_files(etd)
+    etd.links.select { |l| l.match("ucsb_0035") }
+  end
+
+  def self.metadata_file(etd)
+    proquest_files(etd).find { |f| f.match("xml") }
+  end
+
+  def self.dissertation_file(etd)
+    proquest_files(etd).find { |f| f.match("pdf") }
+  end
+
+  def self.supp_files(etd)
+    proquest_files(etd) - [metadata_file(etd), dissertation_file(etd)]
+  end
+
+  def self.metadata_url(etd)
+    Merritt::Feed::HOME + metadata_file(etd)
+  end
+
+  def self.dissertation_url(etd)
+    Merritt::Feed::HOME + dissertation_file(etd)
+  end
+
+  def self.supp_urls(etd)
+    supp_files(etd).map { |f| Merritt::Feed::HOME + f }
+  end
+
+  def self.get_content(url)
+    uri = URI.parse(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    request = Net::HTTP::Get.new(uri.request_uri)
+    request.basic_auth(Rails.application.config_for(:merritt)["merritt_user"],
+                       Rails.application.config_for(:merritt)["merritt_pwd"])
+    response = http.request(request)
+    raise Net::HTTPError.new(response.body, response) if response.code != 200.to_s
+    response.body
+  end
+
+  def self.escape_encodings(url)
+    # double unescape urls to convert encodings
+    # like %252F => %2F => / in urls as shown below
+    # https://merritt.cdlib.org/d/ark:/13030/m5bp580k/9
+    # /producer/Soriano_ucsb_0035N_67/av.jpeg
+    str = CGI.unescape(CGI.unescape(url))
+    str.split("producer/").last
+    str.split("/").last
+  end
+
+  def self.create_proquest_file(file_path, url)
+    File.open(file_path, "wb") do |f|
+      f.write get_content(url)
+    end
+  end
+end