Merge pull request #144 from rails/solid-cache-yml-config

Solid cache yml config

.github/workflows/main.yml

@@ -67,6 +67,4 @@ jobs:
           sleep 2
           bin/rails db:setup
       - name: Run tests
-        run: bin/rails test
-      - name: Run tests (no connects-to)
-        run: NO_CONNECTS_TO=true bin/rails test
+        run: bundle exec rake test

.gitignore

@@ -5,7 +5,7 @@
 /tmp/
 /test/dummy/db/*.sqlite3
 /test/dummy/db/*.sqlite3-*
-/test/dummy/log/*.log
+/test/dummy/log/
 /test/dummy/storage/
 /test/dummy/tmp/
 /test/fixtures/tmp

README.md

@@ -55,12 +55,57 @@ $ bin/rails db:migrate
 
 ### Configuration
 
+#### solid_cache.yml
+
+Configuration will be read from solid_cache.yml. You can change the location of the config file by setting the `SOLID_CACHE_CONFIG` env variable.
+
+The format of the file is:
+
+```yml
+default:
+  store_options: &default_store_options
+    max_age: <%= 60.days.to_i %>
+    namespace: <%= Rails.env %>
+
+development: &production
+  database: development_cache
+  store_options:
+    <<: *default_store_options
+    max_entries: 1_000_000
+
+production: &production
+  databases: [production_cache1, production_cache2]
+  store_options:
+    <<: *default_store_options
+    max_entries: 10_000_000
+```
+
+For the full list of store_options see [Cache configuration](#cache_configuration). Any options passed to the cache lookup will overwrite those specified here.
+
+#### Connection configuration
+
+You can set one of `database`, `databases` and `connects_to` in the config file. They will be used to configure the cache databases in `SolidCache::Record#connects_to`.
+
+Setting `database` to `cache_db` will configure with:
+
+```ruby
+SolidCache::Record.connects_to database: { writing: :cache_db }
+```
+
+Setting `databases` to `[cache_db, cache_db2]` is the equivalent of:
+
+```ruby
+SolidCache::Record.connects_to shards: { cache_db1: { writing: :cache_db1 },  cache_db2: { writing: :cache_db2 } }
+```
+
+If `connects_to` is set it will be passed directly.
+
 #### Engine configuration
 
 There are two options that can be set on the engine:
 
 - `executor` - the [Rails executor](https://guides.rubyonrails.org/threading_and_code_execution.html#executor) used to wrap asynchronous operations, defaults to the app executor
-- `connects_to` - a custom connects to value for the abstract `SolidCache::Record` active record model. Required for sharding and/or using a separate cache database to the main app.
+- `connects_to` - a custom connects to value for the abstract `SolidCache::Record` active record model. Required for sharding and/or using a separate cache database to the main app. This will overwrite any value set in `config/solid_cache.yml`
 
 These can be set in your Rails configuration:
 

Rakefile

@@ -8,3 +8,37 @@ load "rails/tasks/engine.rake"
 load "rails/tasks/statistics.rake"
 
 require "bundler/gem_tasks"
+require "rake/testtask"
+
+def run_without_aborting(*tasks)
+  errors = []
+
+  tasks.each do |task|
+    Rake::Task[task].invoke
+  rescue Exception
+    errors << task
+  end
+
+  abort "Errors running #{errors.join(', ')}" if errors.any?
+end
+
+def configs
+  [ :default, :cluster, :clusters, :clusters_named, :database, :no_database ]
+end
+
+task :test do
+  tasks = configs.map { |config| "test:#{config}" }
+  run_without_aborting(*tasks)
+end
+
+configs.each do |config|
+  namespace :test do
+    task config do
+      if config == :default
+        sh("bin/rails test")
+      else
+        sh("SOLID_CACHE_CONFIG=config/solid_cache_#{config}.yml bin/rails test")
+      end
+    end
+  end
+end

app/models/solid_cache/entry.rb

@@ -10,7 +10,7 @@ class Entry < Record
     VALUE_BYTE_SIZE = 4
     FIXED_SIZE_COLUMNS_BYTE_SIZE = ID_BYTE_SIZE + CREATED_AT_BYTE_SIZE + KEY_HASH_BYTE_SIZE + VALUE_BYTE_SIZE
 
-    self.ignored_columns += [ :key_hash, :byte_size] if SolidCache.key_hash_stage == :ignored
+    self.ignored_columns += [ :key_hash, :byte_size] if SolidCache.configuration.key_hash_stage == :ignored
 
     class << self
       def write(key, value)
@@ -94,12 +94,12 @@ def add_key_hash_and_byte_size(payloads)
         end
 
         def key_hash?
-          @key_hash ||= [ :indexed, :unindexed ].include?(SolidCache.key_hash_stage) &&
+          @key_hash ||= [ :indexed, :unindexed ].include?(SolidCache.configuration.key_hash_stage) &&
             connection.column_exists?(table_name, :key_hash)
         end
 
         def key_hash_indexed?
-          key_hash? && SolidCache.key_hash_stage == :indexed
+          key_hash? && SolidCache.configuration.key_hash_stage == :indexed
         end
 
         def lookup_column

app/models/solid_cache/record.rb

@@ -6,20 +6,32 @@ class Record < ActiveRecord::Base
 
     self.abstract_class = true
 
-    connects_to(**SolidCache.connects_to) if SolidCache.connects_to
+    connects_to(**SolidCache.configuration.connects_to) if SolidCache.configuration.connects_to
 
     class << self
       def disable_instrumentation(&block)
         connection.with_instrumenter(NULL_INSTRUMENTER, &block)
       end
 
       def with_shard(shard, &block)
-        if shard && SolidCache.connects_to
+        if shard && SolidCache.configuration.sharded?
           connected_to(shard: shard, role: default_role, prevent_writes: false, &block)
         else
           block.call
         end
       end
+
+      def each_shard(&block)
+        return to_enum(:each_shard) unless block_given?
+
+        if SolidCache.configuration.sharded?
+          SolidCache.configuration.shard_keys.each do |shard|
+            Record.with_shard(shard, &block)
+          end
+        else
+          yield
+        end
+      end
     end
   end
 end

lib/solid_cache.rb

@@ -9,28 +9,8 @@
 loader.setup
 
 module SolidCache
-  mattr_accessor :executor, :connects_to
-  mattr_accessor :key_hash_stage, default: :indexed
-
-  def self.all_shard_keys
-    all_shards_config&.keys || []
-  end
-
-  def self.all_shards_config
-    connects_to && connects_to[:shards]
-  end
-
-  def self.each_shard(&block)
-    return to_enum(:each_shard) unless block_given?
-
-    if (shards = all_shards_config&.keys)
-      shards.each do |shard|
-        Record.with_shard(shard, &block)
-      end
-    else
-      yield
-    end
-  end
+  mattr_accessor :executor
+  mattr_accessor :configuration, default: Configuration.new
 end
 
 loader.eager_load

lib/solid_cache/cluster.rb

@@ -4,7 +4,10 @@ module SolidCache
   class Cluster
     include Connections, Execution, Expiry, Stats
 
+    attr_reader :error_handler
+
     def initialize(options = {})
+      @error_handler = options[:error_handler]
       super(options)
     end
 

lib/solid_cache/cluster/connections.rb

@@ -3,6 +3,8 @@
 module SolidCache
   class Cluster
     module Connections
+      attr_reader :shard_options
+
       def initialize(options = {})
         super(options)
         @shard_options = options.fetch(:shards, nil)
@@ -40,14 +42,14 @@ def connection_names
         connections.names
       end
 
+      def connections
+        @connections ||= SolidCache::Connections.from_config(@shard_options)
+      end
+
       private
         def setup!
           connections
         end
-
-        def connections
-          @connections ||= SolidCache::Connections.from_config(@shard_options)
-        end
     end
   end
 end

lib/solid_cache/cluster/execution.rb

@@ -19,6 +19,8 @@ def async(&block)
                 instrument(&block)
               end
             end
+          rescue Exception => exception
+            error_handler&.call(method: :async, exception: exception, returning: nil)
           end
         end
 

lib/solid_cache/configuration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module SolidCache
+  class Configuration
+    attr_reader :store_options, :connects_to, :key_hash_stage, :executor
+
+    def initialize(store_options: {}, database: nil, databases: nil, connects_to: nil, key_hash_stage: :indexed, executor: nil)
+      @store_options = store_options
+      @key_hash_stage = key_hash_stage
+      @executor = executor
+      set_connects_to(database: database, databases: databases, connects_to: connects_to)
+    end
+
+    def sharded?
+      connects_to && connects_to[:shards]
+    end
+
+    def shard_keys
+      sharded? ? connects_to[:shards].keys : []
+    end
+
+    private
+      def set_connects_to(database:, databases:, connects_to:)
+        if [database, databases, connects_to].compact.size > 1
+          raise ArgumentError, "You can only specify one of :database, :databases, or :connects_to"
+        end
+
+        @connects_to =
+          case
+          when database
+            { database: { writing: database.to_sym } }
+          when databases
+            { shards: databases.map(&:to_sym).index_with { |database| { writing: database } } }
+          when connects_to
+            connects_to
+          else
+            nil
+          end
+      end
+  end
+end

lib/solid_cache/connections.rb

@@ -3,20 +3,20 @@
 module SolidCache
   module Connections
     def self.from_config(options)
-      if options.present? || SolidCache.all_shards_config.present?
+      if options.present? || SolidCache.configuration.sharded?
         case options
         when NilClass
-          names = SolidCache.all_shard_keys
+          names = SolidCache.configuration.shard_keys
           nodes = names.to_h { |name| [ name, name ] }
         when Array
-          names = options
+          names = options.map(&:to_sym)
           nodes = names.to_h { |name| [ name, name ] }
         when Hash
-          names = options.keys
-          nodes = options.invert
+          names = options.keys.map(&:to_sym)
+          nodes = options.to_h { |names, nodes| [ nodes.to_sym, names.to_sym ] }
         end
 
-        if (unknown_shards = names - SolidCache.all_shard_keys).any?
+        if (unknown_shards = names - SolidCache.configuration.shard_keys).any?
           raise ArgumentError, "Unknown #{"shard".pluralize(unknown_shards)}: #{unknown_shards.join(", ")}"
         end
 

lib/solid_cache/engine.rb

@@ -8,19 +8,21 @@ class Engine < ::Rails::Engine
 
     config.solid_cache = ActiveSupport::OrderedOptions.new
 
-    initializer "solid_cache", before: :run_prepare_callbacks do |app|
-      config.solid_cache.executor ||= app.executor
-
-      SolidCache.executor = config.solid_cache.executor
-      SolidCache.connects_to = config.solid_cache.connects_to
-      if config.solid_cache.key_hash_stage
-        unless [:ignored, :unindexed, :indexed].include?(config.solid_cache.key_hash_stage)
-          raise "ArgumentError, :key_hash_stage must be :ignored, :unindexed or :indexed"
-        end
-        SolidCache.key_hash_stage = config.solid_cache.key_hash_stage
+    initializer "solid_cache.config" do |app|
+      app.paths.add "config/solid_cache", with: ENV["SOLID_CACHE_CONFIG"] || "config/solid_cache.yml"
+
+      if (config_path = Pathname.new(app.config.paths["config/solid_cache"].first)).exist?
+        options = app.config_for(config_path).to_h.deep_symbolize_keys
+        options[:connects_to] = config.solid_cache.connects_to if config.solid_cache.connects_to
+
+        SolidCache.configuration = SolidCache::Configuration.new(**options)
       end
     end
 
+    initializer "solid_cache.app_executor", before: :run_prepare_callbacks do |app|
+      SolidCache.executor = config.solid_cache.executor || app.executor
+    end
+
     config.after_initialize do
       Rails.cache.setup! if Rails.cache.is_a?(Store)
     end

lib/solid_cache/store.rb

@@ -5,6 +5,10 @@ class Store < ActiveSupport::Cache::Store
     include Api, Clusters, Entries, Failsafe
     prepend ActiveSupport::Cache::Strategy::LocalCache
 
+    def initialize(options = {})
+      super(SolidCache.configuration.store_options.merge(options))
+    end
+
     def self.supports_cache_versioning?
       true
     end

lib/solid_cache/store/clusters.rb

@@ -11,7 +11,7 @@ def initialize(options = {})
         clusters_options = options.fetch(:clusters) { [ options.fetch(:cluster, {}) ] }
 
         @clusters = clusters_options.map.with_index do |cluster_options, index|
-          Cluster.new(options.merge(cluster_options).merge(async_writes: index != 0))
+          Cluster.new(options.merge(cluster_options).merge(async_writes: index != 0, error_handler: error_handler))
         end
 
         @primary_cluster = clusters.first

test/dummy/config/application.rb

@@ -24,16 +24,8 @@ class Application < Rails::Application
     # config.time_zone = "Central Time (US & Canada)"
     # config.eager_load_paths << Rails.root.join("extras")
 
-    unless ENV["NO_CONNECTS_TO"]
-      config.solid_cache.connects_to = {
-        shards: {
-          default: { writing: :primary, reading: :primary_replica },
-          primary_shard_one: { writing: :primary_shard_one },
-          primary_shard_two: { writing: :primary_shard_two },
-          secondary_shard_one: { writing: :secondary_shard_one },
-          secondary_shard_two: { writing: :secondary_shard_two }
-        }
-      }
+    initializer :custom_solid_cache_yml, before: :solid_cache do |app|
+      app.paths.add "config/solid_cache", with: ENV["SOLID_CACHE_CONFIG_PATH"]
     end
   end
 end