From 9347dbd0573bb28821ccf7b58b8ae670ac6e7569 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 24 Apr 2025 11:54:31 +0300 Subject: [PATCH 01/27] Feature: Dynamic scheduled tasks --- README.md | 2 + app/models/solid_queue/recurring_task.rb | 1 + lib/solid_queue/configuration.rb | 10 +- lib/solid_queue/scheduler.rb | 6 ++ .../scheduler/recurring_schedule.rb | 46 ++++++-- test/unit/configuration_test.rb | 6 +- test/unit/scheduler_test.rb | 100 +++++++++++++++++- 7 files changed, 152 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b49f9c60..7c13bda3 100644 --- a/README.md +++ b/README.md @@ -707,6 +707,8 @@ Rails.application.config.after_initialize do # or to_prepare end ``` +You can also dynamically add or remove recurring tasks by creating or deleting SolidQueue::RecurringTask records. It works the same way as with static tasks, except you must set the static field to false. Changes won’t be picked up immediately — they take effect after about a one-minute delay. + It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around. **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations. diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index d248f992..d073a277 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -11,6 +11,7 @@ class RecurringTask < Record validate :ensure_existing_job_class scope :static, -> { where(static: true) } + scope :dynamic, -> { where(static: false) } has_many :recurring_executions, foreign_key: :task_key, primary_key: :key diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 94169ca7..da6b0df4 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -103,7 +103,7 @@ def default_options end def invalid_tasks - recurring_tasks.select(&:invalid?) + static_recurring_tasks.select(&:invalid?) end def only_work? @@ -137,8 +137,8 @@ def dispatchers end def schedulers - if !skip_recurring_tasks? && recurring_tasks.any? - [ Process.new(:scheduler, recurring_tasks: recurring_tasks) ] + if !skip_recurring_tasks? + [ Process.new(:scheduler, recurring_tasks: static_recurring_tasks) ] else [] end @@ -154,8 +154,8 @@ def dispatchers_options .map { |options| options.dup.symbolize_keys } end - def recurring_tasks - @recurring_tasks ||= recurring_tasks_config.map do |id, options| + def static_recurring_tasks + @static_recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) end.compact end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 3cec90fa..3ac78d74 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -30,6 +30,12 @@ def run loop do break if shutting_down? + recurring_schedule.update_scheduled_tasks.tap do |updated_tasks| + if updated_tasks.any? + process.update_columns(metadata: metadata.compact) + end + end + interruptible_sleep(SLEEP_INTERVAL) end ensure diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index b765edf1..ea55f9f8 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,10 +4,11 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :configured_tasks, :scheduled_tasks + attr_reader :static_tasks, :configured_tasks, :scheduled_tasks def initialize(tasks) - @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) + @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) + @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new end @@ -17,8 +18,8 @@ def empty? def schedule_tasks wrap_in_app_executor do - persist_tasks - reload_tasks + persist_static_tasks + reload_static_tasks end configured_tasks.each do |task| @@ -26,6 +27,27 @@ def schedule_tasks end end + def dynamic_tasks + SolidQueue::RecurringTask.dynamic + end + + def schedule_new_dynamic_tasks + dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| + schedule_task(task) + end + end + + def unschedule_old_dynamic_tasks + (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| + scheduled_tasks[key].cancel + scheduled_tasks.delete(key) + end + end + + def update_scheduled_tasks + schedule_new_dynamic_tasks + unschedule_old_dynamic_tasks + end + def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -35,18 +57,22 @@ def unschedule_tasks scheduled_tasks.clear end + def static_task_keys + static_tasks.map(&:key) + end + def task_keys - configured_tasks.map(&:key) + static_task_keys + dynamic_tasks.map(&:key) end private - def persist_tasks - SolidQueue::RecurringTask.static.where.not(key: task_keys).delete_all - SolidQueue::RecurringTask.create_or_update_all configured_tasks + def persist_static_tasks + SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all + SolidQueue::RecurringTask.create_or_update_all static_tasks end - def reload_tasks - @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys).to_a + def reload_static_tasks + @static_tasks = SolidQueue::RecurringTask.static.where(key: static_task_keys).to_a end def schedule(task) diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 11c2a5ff..14db95cb 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -21,7 +21,7 @@ class ConfigurationTest < ActiveSupport::TestCase test "default configuration when config given is empty" do configuration = SolidQueue::Configuration.new(config_file: config_file_path(:empty_configuration), recurring_schedule_file: config_file_path(:empty_configuration)) - assert_equal 2, configuration.configured_processes.count + assert_equal 3, configuration.configured_processes.count # includes scheduler for dynamic tasks assert_processes configuration, :worker, 1, queues: "*" assert_processes configuration, :dispatcher, 1, batch_size: SolidQueue::Configuration::DISPATCHER_DEFAULTS[:batch_size] end @@ -134,12 +134,12 @@ class ConfigurationTest < ActiveSupport::TestCase configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_production_only)) assert configuration.valid? - assert_processes configuration, :scheduler, 0 + assert_processes configuration, :scheduler, 1 # Starts in case of dynamic tasks assert_output(/Provided configuration file '[^']+' does not exist\./) do configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_empty)) assert configuration.valid? - assert_processes configuration, :scheduler, 0 + assert_processes configuration, :scheduler, 1 end # No processes diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 3e838c50..21f50f34 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -3,7 +3,7 @@ class SchedulerTest < ActiveSupport::TestCase self.use_transactional_tests = false - test "recurring schedule" do + test "recurring schedule (only static)" do recurring_tasks = { example_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) @@ -17,6 +17,41 @@ class SchedulerTest < ActiveSupport::TestCase scheduler.stop end + test "recurring schedule (only dynamic)" do + SolidQueue::RecurringTask.create( + key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Scheduler", process.kind + + assert_metadata process, recurring_schedule: [ "dynamic_task" ] + ensure + scheduler.stop + end + + test "recurring schedule (static + dynamic)" do + SolidQueue::RecurringTask.create( + key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + + recurring_tasks = { static_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Scheduler", process.kind + + assert_metadata process, recurring_schedule: [ "static_task", "dynamic_task" ] + ensure + scheduler.stop + end + test "run more than one instance of the scheduler with recurring tasks" do recurring_tasks = { example_task: { class: "AddToBufferJob", schedule: "every second", args: 42 } } schedulers = 2.times.collect do @@ -37,4 +72,67 @@ class SchedulerTest < ActiveSupport::TestCase end end end + + test "updates metadata after adding dynamic task post-start" do + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| + s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } + s.start + end + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + # initially there are no recurring_schedule keys + assert process.metadata, {} + + # now create a dynamic task after the scheduler has booted + SolidQueue::RecurringTask.create( + key: "new_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + sleep 1 + + process.reload + + # metadata should now include the new key + assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + ensure + scheduler&.stop + end + + test "updates metadata after removing dynamic task post-start" do + old_dynamic_task = SolidQueue::RecurringTask.create( + key: "old_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| + s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } + s.start + end + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + # initially there is one recurring_schedule key + assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] + + old_dynamic_task.destroy + + sleep 1 + + process.reload + + # The task is unschedule after it's being removed, and it's reflected in the metadata + assert process.metadata, {} + ensure + scheduler&.stop + end end From 3a230875bf16409244abc1ebe01277d63ac3d144 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 15:27:44 +0300 Subject: [PATCH 02/27] Extract polling_interval to scheduler configuration --- .../install/templates/config/queue.yml | 2 ++ lib/solid_queue/configuration.rb | 32 ++++++++++++------- lib/solid_queue/scheduler.rb | 8 ++--- test/unit/scheduler_test.rb | 10 ++---- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/lib/generators/solid_queue/install/templates/config/queue.yml b/lib/generators/solid_queue/install/templates/config/queue.yml index 15691e9d..d7b0e6b9 100644 --- a/lib/generators/solid_queue/install/templates/config/queue.yml +++ b/lib/generators/solid_queue/install/templates/config/queue.yml @@ -7,6 +7,8 @@ default: &default threads: 3 processes: <%%= ENV.fetch("JOB_CONCURRENCY", 1) %> polling_interval: 0.1 + scheduler: + polling_interval: 1 development: <<: *default diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index da6b0df4..244444bd 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -28,6 +28,10 @@ def instantiate concurrency_maintenance_interval: 600 } + SCHEDULER_DEFAULTS = { + polling_interval: 1 + } + DEFAULT_CONFIG_FILE_PATH = "config/queue.yml" DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml" @@ -103,7 +107,7 @@ def default_options end def invalid_tasks - static_recurring_tasks.select(&:invalid?) + recurring_tasks.select(&:invalid?) end def only_work? @@ -137,11 +141,9 @@ def dispatchers end def schedulers - if !skip_recurring_tasks? - [ Process.new(:scheduler, recurring_tasks: static_recurring_tasks) ] - else - [] - end + return [] if skip_recurring_tasks? + + [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] end def workers_options @@ -154,17 +156,25 @@ def dispatchers_options .map { |options| options.dup.symbolize_keys } end - def static_recurring_tasks - @static_recurring_tasks ||= recurring_tasks_config.map do |id, options| + def scheduler_options + @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys + end + + def recurring_tasks + @recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) end.compact end def processes_config @processes_config ||= config_from \ - options.slice(:workers, :dispatchers).presence || options[:config_file], - keys: [ :workers, :dispatchers ], - fallback: { workers: [ WORKER_DEFAULTS ], dispatchers: [ DISPATCHER_DEFAULTS ] } + options.slice(:workers, :dispatchers, :scheduler).presence || options[:config_file], + keys: [ :workers, :dispatchers, :scheduler ], + fallback: { + workers: [ WORKER_DEFAULTS ], + dispatchers: [ DISPATCHER_DEFAULTS ], + scheduler: SCHEDULER_DEFAULTS + } end def recurring_tasks_config diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 3ac78d74..68a72d80 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -5,7 +5,7 @@ class Scheduler < Processes::Base include Processes::Runnable include LifecycleHooks - attr_reader :recurring_schedule + attr_reader :recurring_schedule, :polling_interval after_boot :run_start_hooks after_boot :schedule_recurring_tasks @@ -15,6 +15,8 @@ class Scheduler < Processes::Base def initialize(recurring_tasks:, **options) @recurring_schedule = RecurringSchedule.new(recurring_tasks) + options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS) + @polling_interval = options[:polling_interval] super(**options) end @@ -24,8 +26,6 @@ def metadata end private - SLEEP_INTERVAL = 60 # Right now it doesn't matter, can be set to 1 in the future for dynamic tasks - def run loop do break if shutting_down? @@ -36,7 +36,7 @@ def run end end - interruptible_sleep(SLEEP_INTERVAL) + interruptible_sleep(polling_interval) end ensure SolidQueue.instrument(:shutdown_process, process: self) do diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 21f50f34..1e93681c 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -74,10 +74,7 @@ class SchedulerTest < ActiveSupport::TestCase end test "updates metadata after adding dynamic task post-start" do - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| - s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } - s.start - end + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -113,10 +110,7 @@ class SchedulerTest < ActiveSupport::TestCase arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| - s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } - s.start - end + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) From 3a13962ab8aded3859e5ef928ce1e64c917df8c6 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 16:06:01 +0300 Subject: [PATCH 03/27] Fix abstraction for RecurringSchedule and Process --- lib/solid_queue/processes/registrable.rb | 4 ++++ lib/solid_queue/scheduler.rb | 7 ++---- .../scheduler/recurring_schedule.rb | 22 ++++++++++++++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/solid_queue/processes/registrable.rb b/lib/solid_queue/processes/registrable.rb index cd7769da..7e43cff1 100644 --- a/lib/solid_queue/processes/registrable.rb +++ b/lib/solid_queue/processes/registrable.rb @@ -59,5 +59,9 @@ def heartbeat self.process = nil wake_up end + + def refresh_registered_process + process.update_columns(metadata: metadata.compact) + end end end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 68a72d80..f0464f2b 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -30,11 +30,8 @@ def run loop do break if shutting_down? - recurring_schedule.update_scheduled_tasks.tap do |updated_tasks| - if updated_tasks.any? - process.update_columns(metadata: metadata.compact) - end - end + recurring_schedule.reload! + refresh_registered_process if recurring_schedule.changed? interruptible_sleep(polling_interval) end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index ea55f9f8..65f3be2d 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,12 +4,13 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :static_tasks, :configured_tasks, :scheduled_tasks + attr_reader :static_tasks, :configured_tasks, :scheduled_tasks, :changes def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new + @changes = Concurrent::Hash.new end def empty? @@ -44,10 +45,6 @@ def unschedule_old_dynamic_tasks end end - def update_scheduled_tasks - schedule_new_dynamic_tasks + unschedule_old_dynamic_tasks - end - def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -65,6 +62,21 @@ def task_keys static_task_keys + dynamic_tasks.map(&:key) end + def reload! + { added_tasks: schedule_new_dynamic_tasks, + removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| + if values.any? + changes[key] = values + else + changes.delete(key) + end + end + end + + def changed? + @changes.any? + end + private def persist_static_tasks SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all From d1c0efcef99c048ed34587ee5129dfd6a4c6c4e5 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 17:35:14 +0300 Subject: [PATCH 04/27] Add create and destroy recurring task helpers --- lib/solid_queue.rb | 9 +++++++++ test/solid_queue_test.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index e0d51c8c..0b8ea4e3 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -43,6 +43,15 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor + + def create_recurring_task(key, **attributes) + RecurringTask.create!(**attributes, key:, static: false) + end + + def destroy_recurring_task(id) + RecurringTask.dynamic.find(id).destroy! + end + [ Dispatcher, Scheduler, Worker ].each do |process| define_singleton_method(:"on_#{process.name.demodulize.downcase}_start") do |&block| process.on_start(&block) diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index d6d61b57..2c7bd00b 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -4,4 +4,31 @@ class SolidQueueTest < ActiveSupport::TestCase test "it has a version number" do assert SolidQueue::VERSION end + + test "creates recurring tasks" do + SolidQueue.create_recurring_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue.create_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) + + assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) + assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) + end + + test "destroys recurring tasks" do + dynamic_task = SolidQueue::RecurringTask.create!( + key: "dynamic", command: "puts 'd'", schedule: "every day", static: false + ) + + static_task = SolidQueue::RecurringTask.create!( + key: "static", command: "puts 's'", schedule: "every week", static: true + ) + + SolidQueue.destroy_recurring_task(dynamic_task.id) + + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue.destroy_recurring_task(static_task.id) + end + + assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) + assert SolidQueue::RecurringTask.exists?(key: "static", static: true) + end end From 77ff5231a2a8850e7f052ff5f9a6644fdc36c0ce Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Sat, 12 Jul 2025 12:28:18 +0300 Subject: [PATCH 05/27] Update README with Recurring tasks info --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7c13bda3..6329f377 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,17 @@ It is recommended to set this value less than or equal to the queue database's c - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. +### Scheduler polling interval + +The scheduler process checks for due recurring tasks and reloads dynamic tasks at a configurable interval. You can set this interval using the `polling_interval` key under the `scheduler` section in your `config/queue.yml`: + +```yaml +scheduler: + polling_interval: 5 # seconds +``` + +This controls how frequently the scheduler wakes up to enqueue due recurring jobs and reload dynamic tasks. + ### Queue order and priorities As mentioned above, if you specify a list of queues for a worker, these will be polled in the order given, such as for the list `real_time,background`, no jobs will be taken from `background` unless there aren't any more jobs waiting in `real_time`. @@ -707,8 +718,6 @@ Rails.application.config.after_initialize do # or to_prepare end ``` -You can also dynamically add or remove recurring tasks by creating or deleting SolidQueue::RecurringTask records. It works the same way as with static tasks, except you must set the static field to false. Changes won’t be picked up immediately — they take effect after about a one-minute delay. - It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around. **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations. @@ -734,6 +743,47 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. + +### Creating and Deleting Recurring Tasks Dynamically + +You can create and delete recurring tasks at runtime, without editing the configuration file. Use the following methods: + +#### Creating a recurring task + +```ruby +SolidQueue.schedule_recurring_task( + "my_dynamic_task", + command: "puts 'Hello from a dynamic task!'", + schedule: "every 10 minutes" +) +``` + +This will create a dynamic recurring task with the given key, command, and schedule. You can also use the `class` and `args` options as in the configuration file. + +#### Deleting a recurring task + +```ruby +SolidQueue.delete_recurring_task(task_id) +``` + +This will delete a dynamically scheduled recurring task by its ID. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. + +> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. + +#### Example: Creating and deleting a recurring task + +```ruby +# Create a new dynamic recurring task +recurring_task = SolidQueue.schedule_recurring_task( + "cleanup_temp_files", + command: "TempFileCleaner.clean!", + schedule: "every day at 2am" +) + +# Delete the task later by ID +SolidQueue.delete_recurring_task(recurring_task.id) +``` + ## Inspiration Solid Queue has been inspired by [resque](https://github.com/resque/resque) and [GoodJob](https://github.com/bensheldon/good_job). We recommend checking out these projects as they're great examples from which we've learnt a lot. From f1028a050715c95d94367efe3421d6d270b00f56 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 21 Aug 2025 19:06:33 +0300 Subject: [PATCH 06/27] Use task key instead of id --- lib/solid_queue.rb | 4 ++-- test/solid_queue_test.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 0b8ea4e3..6d088de3 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -48,8 +48,8 @@ def create_recurring_task(key, **attributes) RecurringTask.create!(**attributes, key:, static: false) end - def destroy_recurring_task(id) - RecurringTask.dynamic.find(id).destroy! + def destroy_recurring_task(key) + RecurringTask.dynamic.find_by!(key:).destroy end [ Dispatcher, Scheduler, Worker ].each do |process| diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 2c7bd00b..316add46 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -22,10 +22,10 @@ class SolidQueueTest < ActiveSupport::TestCase key: "static", command: "puts 's'", schedule: "every week", static: true ) - SolidQueue.destroy_recurring_task(dynamic_task.id) + SolidQueue.destroy_recurring_task(dynamic_task.key) assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.destroy_recurring_task(static_task.id) + SolidQueue.destroy_recurring_task(static_task.key) end assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) From 7dd67c88bf8ee83eb916fb8c7c88b4cd60dc7daf Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 21 Aug 2025 19:07:03 +0300 Subject: [PATCH 07/27] Fix mismatches in readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6329f377..adaf419c 100644 --- a/README.md +++ b/README.md @@ -751,7 +751,7 @@ You can create and delete recurring tasks at runtime, without editing the config #### Creating a recurring task ```ruby -SolidQueue.schedule_recurring_task( +SolidQueue.create_recurring_task( "my_dynamic_task", command: "puts 'Hello from a dynamic task!'", schedule: "every 10 minutes" @@ -763,10 +763,10 @@ This will create a dynamic recurring task with the given key, command, and sched #### Deleting a recurring task ```ruby -SolidQueue.delete_recurring_task(task_id) +SolidQueue.destroy_recurring_task(key) ``` -This will delete a dynamically scheduled recurring task by its ID. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. +This will delete a dynamically scheduled recurring task by its key. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. > **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. @@ -774,14 +774,14 @@ This will delete a dynamically scheduled recurring task by its ID. If you attemp ```ruby # Create a new dynamic recurring task -recurring_task = SolidQueue.schedule_recurring_task( +recurring_task = SolidQueue.create_recurring_task( "cleanup_temp_files", command: "TempFileCleaner.clean!", schedule: "every day at 2am" ) -# Delete the task later by ID -SolidQueue.delete_recurring_task(recurring_task.id) +# Delete the task later by key +SolidQueue.destroy_recurring_task("cleanup_temp_files") ``` ## Inspiration From eef747f64f672df7100a7c754dc0f11ac925a12e Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:04:23 +0200 Subject: [PATCH 08/27] Use correct asserts in tests Two assertions were using assert value, message instead of assert_equal/assert_empty, meaning they always passed regardless of the actual metadata content. Fixed to use assert_empty. Also fixed a typo ("unschedule" -> "unscheduled"). --- test/unit/scheduler_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 1e93681c..aac9aa51 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -80,7 +80,7 @@ class SchedulerTest < ActiveSupport::TestCase process = SolidQueue::Process.first # initially there are no recurring_schedule keys - assert process.metadata, {} + assert_empty process.metadata # now create a dynamic task after the scheduler has booted SolidQueue::RecurringTask.create( @@ -124,8 +124,8 @@ class SchedulerTest < ActiveSupport::TestCase process.reload - # The task is unschedule after it's being removed, and it's reflected in the metadata - assert process.metadata, {} + # The task is unscheduled after it's been removed, and it's reflected in the metadata + assert_empty process.metadata ensure scheduler&.stop end From 8e1be8b2b207f8441aff114b2d9da87bd0d82fa4 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:04:45 +0200 Subject: [PATCH 09/27] Add missing platform to Gemfile.lock --- Gemfile.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile.lock b/Gemfile.lock index 772a99da..324821a2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -193,6 +193,7 @@ PLATFORMS arm64-darwin-22 arm64-darwin-23 arm64-darwin-24 + arm64-darwin-25 x86_64-darwin-21 x86_64-darwin-23 x86_64-linux From 5ba7384312207703fad370f3b37e7ac660503ab6 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:08:29 +0200 Subject: [PATCH 10/27] Move some Scheduler::RecurringSchedule methods to private --- .../scheduler/recurring_schedule.rb | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 65f3be2d..5b72d91a 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,7 +4,7 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :static_tasks, :configured_tasks, :scheduled_tasks, :changes + attr_reader :configured_tasks, :scheduled_tasks def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @@ -28,23 +28,6 @@ def schedule_tasks end end - def dynamic_tasks - SolidQueue::RecurringTask.dynamic - end - - def schedule_new_dynamic_tasks - dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| - schedule_task(task) - end - end - - def unschedule_old_dynamic_tasks - (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| - scheduled_tasks[key].cancel - scheduled_tasks.delete(key) - end - end - def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -54,10 +37,6 @@ def unschedule_tasks scheduled_tasks.clear end - def static_task_keys - static_tasks.map(&:key) - end - def task_keys static_task_keys + dynamic_tasks.map(&:key) end @@ -78,6 +57,29 @@ def changed? end private + attr_reader :static_tasks + + def dynamic_tasks + SolidQueue::RecurringTask.dynamic + end + + def static_task_keys + static_tasks.map(&:key) + end + + def schedule_new_dynamic_tasks + dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| + schedule_task(task) + end + end + + def unschedule_old_dynamic_tasks + (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| + scheduled_tasks[key].cancel + scheduled_tasks.delete(key) + end + end + def persist_static_tasks SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all SolidQueue::RecurringTask.create_or_update_all static_tasks From dd67d58d133b28e2000825d3bcd66462131b4040 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:09:26 +0200 Subject: [PATCH 11/27] Wrap reload in app executor DB queries in reload! (dynamic_tasks.where.not(...), RecurringTask.pluck(:key)) were not wrapped in the app executor, which could cause connection management issues. Wrapped in wrap_in_app_executor. --- lib/solid_queue/scheduler/recurring_schedule.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 5b72d91a..2dacada0 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -42,12 +42,14 @@ def task_keys end def reload! - { added_tasks: schedule_new_dynamic_tasks, - removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| - if values.any? - changes[key] = values - else - changes.delete(key) + wrap_in_app_executor do + { added_tasks: schedule_new_dynamic_tasks, + removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| + if values.any? + @changes[key] = values + else + @changes.delete(key) + end end end end From b90cce501ee8a687228ebe51e6cf4994bb233718 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:10:39 +0200 Subject: [PATCH 12/27] Fix empty? method in Scheduler::RecurringSchedule empty? checked stale configured_tasks (lib/solid_queue/scheduler/recurring_schedule.rb) -- configured_tasks was set once in initialize and never updated with dynamic tasks. This meant empty? could return true even when dynamic tasks existed, causing the scheduler to exit prematurely in inline mode. Changed empty? to check scheduled_tasks.empty? && dynamic_tasks.none?. --- lib/solid_queue/scheduler/recurring_schedule.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 2dacada0..ff7e7a7b 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -14,7 +14,7 @@ def initialize(tasks) end def empty? - configured_tasks.empty? + scheduled_tasks.empty? && dynamic_tasks.none? end def schedule_tasks From 6b81c9571131d9c9aed59097ebdfe72718ac3cba Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:13:21 +0200 Subject: [PATCH 13/27] Prevent stale change by adding clear_changes in scheduler's loop Changes not cleared after consumption (lib/solid_queue/scheduler.rb, lib/solid_queue/scheduler/recurring_schedule.rb) -- Added clear_changes method and call it in the scheduler's run loop after refresh_registered_process, preventing stale change state from persisting. --- lib/solid_queue/scheduler.rb | 6 +++++- lib/solid_queue/scheduler/recurring_schedule.rb | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index f0464f2b..920c5d3a 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -31,7 +31,11 @@ def run break if shutting_down? recurring_schedule.reload! - refresh_registered_process if recurring_schedule.changed? + + if recurring_schedule.changed? + refresh_registered_process + recurring_schedule.clear_changes + end interruptible_sleep(polling_interval) end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index ff7e7a7b..dfe192f7 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -58,6 +58,10 @@ def changed? @changes.any? end + def clear_changes + @changes.clear + end + private attr_reader :static_tasks From 99ba202295a58f02e84fdc7ecf207a9119a664ef Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:13:43 +0200 Subject: [PATCH 14/27] Add test for enqueuing the job --- test/unit/scheduler_test.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index aac9aa51..669afdcf 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -73,6 +73,22 @@ class SchedulerTest < ActiveSupport::TestCase end end + test "dynamic task actually enqueues jobs" do + SolidQueue::RecurringTask.create!( + key: "dynamic_enqueue_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + sleep 2 + + assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" + assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + ensure + scheduler&.stop + end + test "updates metadata after adding dynamic task post-start" do scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) From 79fca8de657fb4862b13f5dbbe8501e7737d6ffb Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:18:54 +0200 Subject: [PATCH 15/27] Improve create_recurring_task - use RecurringTask .from_configuration --- README.md | 9 +++++---- lib/solid_queue.rb | 7 +++++-- test/solid_queue_test.rb | 9 +++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index adaf419c..ed00d7dc 100644 --- a/README.md +++ b/README.md @@ -753,12 +753,13 @@ You can create and delete recurring tasks at runtime, without editing the config ```ruby SolidQueue.create_recurring_task( "my_dynamic_task", - command: "puts 'Hello from a dynamic task!'", + class: "MyJob", + args: [1, 2], schedule: "every 10 minutes" ) ``` -This will create a dynamic recurring task with the given key, command, and schedule. You can also use the `class` and `args` options as in the configuration file. +This will create a dynamic recurring task with the given key, class, and schedule. The API accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. #### Deleting a recurring task @@ -774,9 +775,9 @@ This will delete a dynamically scheduled recurring task by its key. If you attem ```ruby # Create a new dynamic recurring task -recurring_task = SolidQueue.create_recurring_task( +SolidQueue.create_recurring_task( "cleanup_temp_files", - command: "TempFileCleaner.clean!", + class: "TempFileCleanerJob", schedule: "every day at 2am" ) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 6d088de3..7ca1fd03 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,8 +44,11 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def create_recurring_task(key, **attributes) - RecurringTask.create!(**attributes, key:, static: false) + def create_recurring_task(key, **options) + RecurringTask.from_configuration(key, **options).tap do |task| + task.static = false + task.save! + end end def destroy_recurring_task(key) diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 316add46..01c41a19 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -13,6 +13,15 @@ class SolidQueueTest < ActiveSupport::TestCase assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) end + test "creates recurring tasks with class and args (same keys as YAML config)" do + SolidQueue.create_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + + task = SolidQueue::RecurringTask.find_by!(key: "test 3") + assert_equal "AddToBufferJob", task.class_name + assert_equal [ 42 ], task.arguments + assert_equal false, task.static + end + test "destroys recurring tasks" do dynamic_task = SolidQueue::RecurringTask.create!( key: "dynamic", command: "puts 'd'", schedule: "every day", static: false From 7ca2eb7a54de1f7f6c5e66eef86e0e0d24c71036 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:30:06 +0200 Subject: [PATCH 16/27] Fix names for public methods --- README.md | 26 +++++++++++++------------- lib/solid_queue.rb | 4 ++-- test/solid_queue_test.rb | 16 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ed00d7dc..63019c65 100644 --- a/README.md +++ b/README.md @@ -744,14 +744,14 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. -### Creating and Deleting Recurring Tasks Dynamically +### Scheduling and Unscheduling Recurring Tasks Dynamically -You can create and delete recurring tasks at runtime, without editing the configuration file. Use the following methods: +You can schedule and unschedule recurring tasks at runtime, without editing the configuration file. Use the following methods: -#### Creating a recurring task +#### Scheduling a recurring task ```ruby -SolidQueue.create_recurring_task( +SolidQueue.schedule_task( "my_dynamic_task", class: "MyJob", args: [1, 2], @@ -761,28 +761,28 @@ SolidQueue.create_recurring_task( This will create a dynamic recurring task with the given key, class, and schedule. The API accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. -#### Deleting a recurring task +#### Unscheduling a recurring task ```ruby -SolidQueue.destroy_recurring_task(key) +SolidQueue.unschedule_task(key) ``` -This will delete a dynamically scheduled recurring task by its key. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. +This will delete a dynamically scheduled recurring task by its key. If you attempt to unschedule a static (configuration-defined) recurring task, an error will be raised. -> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. +> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be unscheduled at runtime. Attempting to do so will raise an error. -#### Example: Creating and deleting a recurring task +#### Example: Scheduling and unscheduling a recurring task ```ruby -# Create a new dynamic recurring task -SolidQueue.create_recurring_task( +# Schedule a new dynamic recurring task +SolidQueue.schedule_task( "cleanup_temp_files", class: "TempFileCleanerJob", schedule: "every day at 2am" ) -# Delete the task later by key -SolidQueue.destroy_recurring_task("cleanup_temp_files") +# Unschedule the task later by key +SolidQueue.unschedule_task("cleanup_temp_files") ``` ## Inspiration diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 7ca1fd03..11994b76 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,14 +44,14 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def create_recurring_task(key, **options) + def schedule_task(key, **options) RecurringTask.from_configuration(key, **options).tap do |task| task.static = false task.save! end end - def destroy_recurring_task(key) + def unschedule_task(key) RecurringTask.dynamic.find_by!(key:).destroy end diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 01c41a19..09140654 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -5,16 +5,16 @@ class SolidQueueTest < ActiveSupport::TestCase assert SolidQueue::VERSION end - test "creates recurring tasks" do - SolidQueue.create_recurring_task("test 1", command: "puts 1", schedule: "every hour") - SolidQueue.create_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) + test "schedules recurring tasks" do + SolidQueue.schedule_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue.schedule_task("test 2", command: "puts 2", schedule: "every minute", static: true) assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) end - test "creates recurring tasks with class and args (same keys as YAML config)" do - SolidQueue.create_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + test "schedules recurring tasks with class and args (same keys as YAML config)" do + SolidQueue.schedule_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") task = SolidQueue::RecurringTask.find_by!(key: "test 3") assert_equal "AddToBufferJob", task.class_name @@ -22,7 +22,7 @@ class SolidQueueTest < ActiveSupport::TestCase assert_equal false, task.static end - test "destroys recurring tasks" do + test "unschedules recurring tasks" do dynamic_task = SolidQueue::RecurringTask.create!( key: "dynamic", command: "puts 'd'", schedule: "every day", static: false ) @@ -31,10 +31,10 @@ class SolidQueueTest < ActiveSupport::TestCase key: "static", command: "puts 's'", schedule: "every week", static: true ) - SolidQueue.destroy_recurring_task(dynamic_task.key) + SolidQueue.unschedule_task(dynamic_task.key) assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.destroy_recurring_task(static_task.key) + SolidQueue.unschedule_task(static_task.key) end assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) From 890dabe4f00ba3077c123ad95024077a70ab4b94 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:29:02 +0200 Subject: [PATCH 17/27] Prevent stale configured_tasks value --- lib/solid_queue/scheduler/recurring_schedule.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index dfe192f7..12c7af3d 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,15 +4,18 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :configured_tasks, :scheduled_tasks + attr_reader :scheduled_tasks def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) - @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new @changes = Concurrent::Hash.new end + def configured_tasks + static_tasks + dynamic_tasks.to_a + end + def empty? scheduled_tasks.empty? && dynamic_tasks.none? end @@ -38,7 +41,7 @@ def unschedule_tasks end def task_keys - static_task_keys + dynamic_tasks.map(&:key) + static_task_keys + dynamic_tasks.pluck(:key) end def reload! From abbf54ed7b25f3592ce069601f59c626e74960c9 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:29:40 +0200 Subject: [PATCH 18/27] Use wrap_in_app_executor for refresh_registered_process --- lib/solid_queue/processes/registrable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/solid_queue/processes/registrable.rb b/lib/solid_queue/processes/registrable.rb index 7e43cff1..94e9ed77 100644 --- a/lib/solid_queue/processes/registrable.rb +++ b/lib/solid_queue/processes/registrable.rb @@ -61,7 +61,7 @@ def heartbeat end def refresh_registered_process - process.update_columns(metadata: metadata.compact) + wrap_in_app_executor { process&.update_columns(metadata: metadata.compact) } end end end From 4ec36d6dfd34cde414d09a105cf74a1c0b826cbc Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:31:58 +0200 Subject: [PATCH 19/27] Update tests and README --- README.md | 2 + lib/solid_queue/configuration.rb | 3 ++ test/solid_queue_test.rb | 14 ++++++ test/unit/scheduler_test.rb | 74 ++++++++++++++++---------------- 4 files changed, 57 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 63019c65..536c2f86 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,8 @@ scheduler: This controls how frequently the scheduler wakes up to enqueue due recurring jobs and reload dynamic tasks. +> **Note:** The scheduler process always starts by default to support dynamic recurring tasks, even if no static tasks are configured in `config/recurring.yml`. If you don't use recurring tasks at all, you can disable the scheduler by setting `SOLID_QUEUE_SKIP_RECURRING=true` or passing `skip_recurring: true` in the configuration. + ### Queue order and priorities As mentioned above, if you specify a list of queues for a worker, these will be polled in the order given, such as for the list `real_time,background`, no jobs will be taken from `background` unless there aren't any more jobs waiting in `real_time`. diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 244444bd..21f7ad5d 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -143,6 +143,9 @@ def dispatchers def schedulers return [] if skip_recurring_tasks? + # Always start a scheduler (even with no static recurring tasks) to support + # dynamic tasks that may be added at runtime via SolidQueue.schedule_task. + # Use skip_recurring: true or SOLID_QUEUE_SKIP_RECURRING=true to disable. [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] end diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 09140654..542c3d82 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -40,4 +40,18 @@ class SolidQueueTest < ActiveSupport::TestCase assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) assert SolidQueue::RecurringTask.exists?(key: "static", static: true) end + + test "schedule_task with duplicate key raises error" do + SolidQueue.schedule_task("duplicate_test", command: "puts 1", schedule: "every hour") + + assert_raises(ActiveRecord::RecordNotUnique) do + SolidQueue.schedule_task("duplicate_test", command: "puts 2", schedule: "every minute") + end + end + + test "unschedule_task with nonexistent key raises RecordNotFound" do + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue.unschedule_task("nonexistent_key") + end + end end diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 669afdcf..882dcd72 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -81,10 +81,12 @@ class SchedulerTest < ActiveSupport::TestCase scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) - sleep 2 + wait_while_with_timeout(3.seconds) { SolidQueue::Job.count < 1 } - assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" - assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + skip_active_record_query_cache do + assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" + assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + end ensure scheduler&.stop end @@ -94,54 +96,54 @@ class SchedulerTest < ActiveSupport::TestCase wait_for_registered_processes(1, timeout: 1.second) - process = SolidQueue::Process.first - # initially there are no recurring_schedule keys - assert_empty process.metadata - - # now create a dynamic task after the scheduler has booted - SolidQueue::RecurringTask.create( - key: "new_dynamic_task", - static: false, - class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] - ) - - sleep 1 - - process.reload - - # metadata should now include the new key - assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + skip_active_record_query_cache do + process = SolidQueue::Process.first + # initially there are no recurring_schedule keys + assert_empty process.metadata + + # now create a dynamic task after the scheduler has booted + SolidQueue::RecurringTask.create!( + key: "new_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + wait_while_with_timeout(3.seconds) { process.reload.metadata.empty? } + + # metadata should now include the new key + assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + end ensure scheduler&.stop end test "updates metadata after removing dynamic task post-start" do - old_dynamic_task = SolidQueue::RecurringTask.create( - key: "old_dynamic_task", - static: false, + old_dynamic_task = SolidQueue::RecurringTask.create!( + key: "old_dynamic_task", + static: false, class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] + schedule: "every second", + arguments: [ 42 ] ) scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) - process = SolidQueue::Process.first - # initially there is one recurring_schedule key - assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] - - old_dynamic_task.destroy + skip_active_record_query_cache do + process = SolidQueue::Process.first + # initially there is one recurring_schedule key + assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] - sleep 1 + old_dynamic_task.destroy - process.reload + wait_while_with_timeout(3.seconds) { process.reload.metadata.present? } - # The task is unscheduled after it's been removed, and it's reflected in the metadata - assert_empty process.metadata + # The task is unscheduled after it's been removed, and it's reflected in the metadata + assert_empty process.metadata + end ensure scheduler&.stop end From e1c1601051e7dbef026351d5824a85d46cf34f6e Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Tue, 24 Feb 2026 22:05:18 +0100 Subject: [PATCH 20/27] Fix style to match project conventions Remove extra blank lines, column-aligned hashes, and inline comments on test assertions. Revert unrelated Gemfile.lock platform addition. Filter :static from schedule_task options since dynamic tasks are always non-static by definition. Co-Authored-By: Claude Opus 4.6 --- Gemfile.lock | 1 - README.md | 2 -- lib/solid_queue.rb | 3 +-- lib/solid_queue/configuration.rb | 1 - test/unit/configuration_test.rb | 4 ++-- test/unit/scheduler_test.rb | 25 +++++++++---------------- 6 files changed, 12 insertions(+), 24 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 324821a2..772a99da 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -193,7 +193,6 @@ PLATFORMS arm64-darwin-22 arm64-darwin-23 arm64-darwin-24 - arm64-darwin-25 x86_64-darwin-21 x86_64-darwin-23 x86_64-linux diff --git a/README.md b/README.md index 536c2f86..9238edd7 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,6 @@ It is recommended to set this value less than or equal to the queue database's c - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting. **Note**: this option will be ignored if [running in `async` mode](#fork-vs-async-mode). - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. - ### Scheduler polling interval The scheduler process checks for due recurring tasks and reloads dynamic tasks at a configurable interval. You can set this interval using the `polling_interval` key under the `scheduler` section in your `config/queue.yml`: @@ -745,7 +744,6 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. - ### Scheduling and Unscheduling Recurring Tasks Dynamically You can schedule and unschedule recurring tasks at runtime, without editing the configuration file. Use the following methods: diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 11994b76..27c8345d 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -43,9 +43,8 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def schedule_task(key, **options) - RecurringTask.from_configuration(key, **options).tap do |task| + RecurringTask.from_configuration(key, **options.except(:static)).tap do |task| task.static = false task.save! end diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 21f7ad5d..4d0ce72e 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -186,7 +186,6 @@ def recurring_tasks_config end end - def config_from(file_or_hash, keys: [], fallback: {}, env: Rails.env) load_config_from(file_or_hash).then do |config| config = config[env.to_sym] ? config[env.to_sym] : config diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 14db95cb..aed0c18c 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -21,7 +21,7 @@ class ConfigurationTest < ActiveSupport::TestCase test "default configuration when config given is empty" do configuration = SolidQueue::Configuration.new(config_file: config_file_path(:empty_configuration), recurring_schedule_file: config_file_path(:empty_configuration)) - assert_equal 3, configuration.configured_processes.count # includes scheduler for dynamic tasks + assert_equal 3, configuration.configured_processes.count assert_processes configuration, :worker, 1, queues: "*" assert_processes configuration, :dispatcher, 1, batch_size: SolidQueue::Configuration::DISPATCHER_DEFAULTS[:batch_size] end @@ -134,7 +134,7 @@ class ConfigurationTest < ActiveSupport::TestCase configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_production_only)) assert configuration.valid? - assert_processes configuration, :scheduler, 1 # Starts in case of dynamic tasks + assert_processes configuration, :scheduler, 1 assert_output(/Provided configuration file '[^']+' does not exist\./) do configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_empty)) diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 882dcd72..f164bc1b 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -21,7 +21,7 @@ class SchedulerTest < ActiveSupport::TestCase SolidQueue::RecurringTask.create( key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -98,21 +98,17 @@ class SchedulerTest < ActiveSupport::TestCase skip_active_record_query_cache do process = SolidQueue::Process.first - # initially there are no recurring_schedule keys assert_empty process.metadata - # now create a dynamic task after the scheduler has booted SolidQueue::RecurringTask.create!( - key: "new_dynamic_task", - static: false, + key: "new_dynamic_task", + static: false, class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] + schedule: "every second", + arguments: [ 42 ] ) wait_while_with_timeout(3.seconds) { process.reload.metadata.empty? } - - # metadata should now include the new key assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] end ensure @@ -121,11 +117,11 @@ class SchedulerTest < ActiveSupport::TestCase test "updates metadata after removing dynamic task post-start" do old_dynamic_task = SolidQueue::RecurringTask.create!( - key: "old_dynamic_task", - static: false, + key: "old_dynamic_task", + static: false, class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] + schedule: "every second", + arguments: [ 42 ] ) scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) @@ -134,14 +130,11 @@ class SchedulerTest < ActiveSupport::TestCase skip_active_record_query_cache do process = SolidQueue::Process.first - # initially there is one recurring_schedule key assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] old_dynamic_task.destroy wait_while_with_timeout(3.seconds) { process.reload.metadata.present? } - - # The task is unscheduled after it's been removed, and it's reflected in the metadata assert_empty process.metadata end ensure From 2f1655833968776b5c6f76fef8b29d5d295890c3 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 14:35:36 +0100 Subject: [PATCH 21/27] Make dynamic recurring tasks opt-in and rename public API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dynamic tasks require explicit `dynamic_tasks: true` in the scheduler config. Without it, the scheduler behaves as before — no extra DB queries, no polling for dynamic changes. Default polling interval changed from 1s to 5s. Rename schedule_task/unschedule_task to schedule_recurring_task/ unschedule_recurring_task so the API clearly communicates these are recurring tasks. Co-Authored-By: Claude Opus 4.6 --- README.md | 58 +++++++++---------- .../install/templates/config/queue.yml | 2 - lib/solid_queue.rb | 4 +- lib/solid_queue/configuration.rb | 16 +++-- lib/solid_queue/scheduler.rb | 24 ++++++-- .../scheduler/recurring_schedule.rb | 25 ++++++-- test/solid_queue_test.rb | 20 +++---- test/unit/configuration_test.rb | 6 +- test/unit/scheduler_test.rb | 10 ++-- 9 files changed, 99 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 9238edd7..ecb67501 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Solid Queue can be used with SQL databases such as MySQL, PostgreSQL, or SQLite, - [Workers, dispatchers, and scheduler](#workers-dispatchers-and-scheduler) - [Fork vs. async mode](#fork-vs-async-mode) - [Configuration](#configuration) + - [Optional scheduler configuration](#optional-scheduler-configuration) - [Queue order and priorities](#queue-order-and-priorities) - [Queues specification and performance](#queues-specification-and-performance) - [Threads, processes, and signals](#threads-processes-and-signals) @@ -31,6 +32,7 @@ Solid Queue can be used with SQL databases such as MySQL, PostgreSQL, or SQLite, - [Puma plugin](#puma-plugin) - [Jobs and transactional integrity](#jobs-and-transactional-integrity) - [Recurring tasks](#recurring-tasks) + - [Scheduling and unscheduling recurring tasks dynamically](#scheduling-and-unscheduling-recurring-tasks-dynamically) - [Inspiration](#inspiration) - [License](#license) @@ -209,7 +211,7 @@ By default, Solid Queue will try to find your configuration under `config/queue. bin/jobs -c config/calendar.yml ``` -You can also skip all recurring tasks by setting the environment variable `SOLID_QUEUE_SKIP_RECURRING=true`. This is useful for environments like staging, review apps, or development where you don't want any recurring jobs to run. This is equivalent to using the `--skip-recurring` option with `bin/jobs`. +You can also skip the scheduler process by setting the environment variable `SOLID_QUEUE_SKIP_RECURRING=true`. This is useful for environments like staging, review apps, or development where you don't want any recurring jobs to run. This is equivalent to using the `--skip-recurring` option with `bin/jobs`. This is what this configuration looks like: @@ -227,6 +229,10 @@ production: threads: 5 polling_interval: 0.1 processes: 3 + scheduler: + dynamic_tasks_enabled: true + polling_interval: 5 + ``` Everything is optional. If no configuration at all is provided, Solid Queue will run with one dispatcher and one worker with default settings. If you want to run only dispatchers or workers, you just need to include that section alone in the configuration. For example, with the following configuration: @@ -270,18 +276,19 @@ It is recommended to set this value less than or equal to the queue database's c - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting. **Note**: this option will be ignored if [running in `async` mode](#fork-vs-async-mode). - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. -### Scheduler polling interval -The scheduler process checks for due recurring tasks and reloads dynamic tasks at a configurable interval. You can set this interval using the `polling_interval` key under the `scheduler` section in your `config/queue.yml`: +### Optional scheduler configuration + +Optionally, you can configure the scheduler process under the `scheduler` section in your `config/queue.yml` if you'd like to [schedule recurring tasks dynamically](#scheduling-and-unscheduling-recurring-tasks-dynamically). ```yaml scheduler: - polling_interval: 5 # seconds + dynamic_tasks_enabled: true + polling_interval: 5 ``` -This controls how frequently the scheduler wakes up to enqueue due recurring jobs and reload dynamic tasks. - -> **Note:** The scheduler process always starts by default to support dynamic recurring tasks, even if no static tasks are configured in `config/recurring.yml`. If you don't use recurring tasks at all, you can disable the scheduler by setting `SOLID_QUEUE_SKIP_RECURRING=true` or passing `skip_recurring: true` in the configuration. +- `dynamic_tasks_enabled`: whether the scheduler should poll for [dynamically scheduled recurring tasks](#scheduling-and-unscheduling-recurring-tasks-dynamically). This is `false` by default. When enabled, the scheduler will poll the database at the given `polling_interval` to pick up tasks scheduled via `SolidQueue.schedule_recurring_task`. +- `polling_interval`: how frequently (in seconds) the scheduler checks for dynamic task changes. Defaults to `5`. ### Queue order and priorities @@ -744,14 +751,19 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. -### Scheduling and Unscheduling Recurring Tasks Dynamically +### Scheduling and unscheduling recurring tasks dynamically + +You can schedule and unschedule recurring tasks at runtime, without editing the configuration file. To enable this, you need to set `dynamic_tasks_enabled: true` in the `scheduler` section of your `config/queue.yml`, [as explained earlier](#optional-scheduler-configuration). -You can schedule and unschedule recurring tasks at runtime, without editing the configuration file. Use the following methods: +```yaml +scheduler: + dynamic_tasks_enabled: true +``` -#### Scheduling a recurring task +Then you can use the following methods to add recurring tasks dynamically: ```ruby -SolidQueue.schedule_task( +SolidQueue.schedule_recurring_task( "my_dynamic_task", class: "MyJob", args: [1, 2], @@ -759,31 +771,17 @@ SolidQueue.schedule_task( ) ``` -This will create a dynamic recurring task with the given key, class, and schedule. The API accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. +This accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. -#### Unscheduling a recurring task +To remove a dynamically scheduled task: ```ruby -SolidQueue.unschedule_task(key) +SolidQueue.unschedule_recurring_task("my_dynamic_task") ``` -This will delete a dynamically scheduled recurring task by its key. If you attempt to unschedule a static (configuration-defined) recurring task, an error will be raised. +Only dynamic tasks can be unscheduled at runtime. Attempting to unschedule a static task (defined in `config/recurring.yml`) will raise an `ActiveRecord::RecordNotFound` error. -> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be unscheduled at runtime. Attempting to do so will raise an error. - -#### Example: Scheduling and unscheduling a recurring task - -```ruby -# Schedule a new dynamic recurring task -SolidQueue.schedule_task( - "cleanup_temp_files", - class: "TempFileCleanerJob", - schedule: "every day at 2am" -) - -# Unschedule the task later by key -SolidQueue.unschedule_task("cleanup_temp_files") -``` +Tasks scheduled like this persist between Solid Queue's restarts and won't stop running until you manually unschedule them. ## Inspiration diff --git a/lib/generators/solid_queue/install/templates/config/queue.yml b/lib/generators/solid_queue/install/templates/config/queue.yml index d7b0e6b9..15691e9d 100644 --- a/lib/generators/solid_queue/install/templates/config/queue.yml +++ b/lib/generators/solid_queue/install/templates/config/queue.yml @@ -7,8 +7,6 @@ default: &default threads: 3 processes: <%%= ENV.fetch("JOB_CONCURRENCY", 1) %> polling_interval: 0.1 - scheduler: - polling_interval: 1 development: <<: *default diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 27c8345d..af4b5a84 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -43,14 +43,14 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def schedule_task(key, **options) + def schedule_recurring_task(key, **options) RecurringTask.from_configuration(key, **options.except(:static)).tap do |task| task.static = false task.save! end end - def unschedule_task(key) + def unschedule_recurring_task(key) RecurringTask.dynamic.find_by!(key:).destroy end diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 4d0ce72e..d0a46265 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -29,7 +29,8 @@ def instantiate } SCHEDULER_DEFAULTS = { - polling_interval: 1 + polling_interval: 5, + dynamic_tasks: false } DEFAULT_CONFIG_FILE_PATH = "config/queue.yml" @@ -143,10 +144,11 @@ def dispatchers def schedulers return [] if skip_recurring_tasks? - # Always start a scheduler (even with no static recurring tasks) to support - # dynamic tasks that may be added at runtime via SolidQueue.schedule_task. - # Use skip_recurring: true or SOLID_QUEUE_SKIP_RECURRING=true to disable. - [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] + if recurring_tasks.any? || dynamic_tasks_enabled? + [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] + else + [] + end end def workers_options @@ -163,6 +165,10 @@ def scheduler_options @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys end + def dynamic_tasks_enabled? + scheduler_options.fetch(:dynamic_tasks, SCHEDULER_DEFAULTS[:dynamic_tasks]) + end + def recurring_tasks @recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 920c5d3a..c224f2b8 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -14,9 +14,10 @@ class Scheduler < Processes::Base after_shutdown :run_exit_hooks def initialize(recurring_tasks:, **options) - @recurring_schedule = RecurringSchedule.new(recurring_tasks) options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS) + @dynamic_tasks = options[:dynamic_tasks] @polling_interval = options[:polling_interval] + @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks: @dynamic_tasks) super(**options) end @@ -26,7 +27,24 @@ def metadata end private + attr_reader :dynamic_tasks + def run + if dynamic_tasks + poll_for_dynamic_tasks + else + loop do + break if shutting_down? + interruptible_sleep(polling_interval) + end + end + ensure + SolidQueue.instrument(:shutdown_process, process: self) do + run_callbacks(:shutdown) { shutdown } + end + end + + def poll_for_dynamic_tasks loop do break if shutting_down? @@ -39,10 +57,6 @@ def run interruptible_sleep(polling_interval) end - ensure - SolidQueue.instrument(:shutdown_process, process: self) do - run_callbacks(:shutdown) { shutdown } - end end def schedule_recurring_tasks diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 12c7af3d..8b558271 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -6,18 +6,27 @@ class Scheduler::RecurringSchedule attr_reader :scheduled_tasks - def initialize(tasks) + def initialize(tasks, dynamic_tasks: false) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @scheduled_tasks = Concurrent::Hash.new @changes = Concurrent::Hash.new + @dynamic_tasks_enabled = dynamic_tasks end def configured_tasks - static_tasks + dynamic_tasks.to_a + if dynamic_tasks_enabled? + static_tasks + dynamic_tasks.to_a + else + static_tasks + end end def empty? - scheduled_tasks.empty? && dynamic_tasks.none? + if dynamic_tasks_enabled? + scheduled_tasks.empty? && dynamic_tasks.none? + else + scheduled_tasks.empty? + end end def schedule_tasks @@ -41,7 +50,11 @@ def unschedule_tasks end def task_keys - static_task_keys + dynamic_tasks.pluck(:key) + if dynamic_tasks_enabled? + static_task_keys + dynamic_tasks.pluck(:key) + else + static_task_keys + end end def reload! @@ -68,6 +81,10 @@ def clear_changes private attr_reader :static_tasks + def dynamic_tasks_enabled? + @dynamic_tasks_enabled + end + def dynamic_tasks SolidQueue::RecurringTask.dynamic end diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 542c3d82..768b9c3c 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -6,15 +6,15 @@ class SolidQueueTest < ActiveSupport::TestCase end test "schedules recurring tasks" do - SolidQueue.schedule_task("test 1", command: "puts 1", schedule: "every hour") - SolidQueue.schedule_task("test 2", command: "puts 2", schedule: "every minute", static: true) + SolidQueue.schedule_recurring_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue.schedule_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) end test "schedules recurring tasks with class and args (same keys as YAML config)" do - SolidQueue.schedule_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + SolidQueue.schedule_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") task = SolidQueue::RecurringTask.find_by!(key: "test 3") assert_equal "AddToBufferJob", task.class_name @@ -31,27 +31,27 @@ class SolidQueueTest < ActiveSupport::TestCase key: "static", command: "puts 's'", schedule: "every week", static: true ) - SolidQueue.unschedule_task(dynamic_task.key) + SolidQueue.unschedule_recurring_task(dynamic_task.key) assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.unschedule_task(static_task.key) + SolidQueue.unschedule_recurring_task(static_task.key) end assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) assert SolidQueue::RecurringTask.exists?(key: "static", static: true) end - test "schedule_task with duplicate key raises error" do - SolidQueue.schedule_task("duplicate_test", command: "puts 1", schedule: "every hour") + test "schedule_recurring_task with duplicate key raises error" do + SolidQueue.schedule_recurring_task("duplicate_test", command: "puts 1", schedule: "every hour") assert_raises(ActiveRecord::RecordNotUnique) do - SolidQueue.schedule_task("duplicate_test", command: "puts 2", schedule: "every minute") + SolidQueue.schedule_recurring_task("duplicate_test", command: "puts 2", schedule: "every minute") end end - test "unschedule_task with nonexistent key raises RecordNotFound" do + test "unschedule_recurring_task with nonexistent key raises RecordNotFound" do assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.unschedule_task("nonexistent_key") + SolidQueue.unschedule_recurring_task("nonexistent_key") end end end diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index aed0c18c..11c2a5ff 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -21,7 +21,7 @@ class ConfigurationTest < ActiveSupport::TestCase test "default configuration when config given is empty" do configuration = SolidQueue::Configuration.new(config_file: config_file_path(:empty_configuration), recurring_schedule_file: config_file_path(:empty_configuration)) - assert_equal 3, configuration.configured_processes.count + assert_equal 2, configuration.configured_processes.count assert_processes configuration, :worker, 1, queues: "*" assert_processes configuration, :dispatcher, 1, batch_size: SolidQueue::Configuration::DISPATCHER_DEFAULTS[:batch_size] end @@ -134,12 +134,12 @@ class ConfigurationTest < ActiveSupport::TestCase configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_production_only)) assert configuration.valid? - assert_processes configuration, :scheduler, 1 + assert_processes configuration, :scheduler, 0 assert_output(/Provided configuration file '[^']+' does not exist\./) do configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_empty)) assert configuration.valid? - assert_processes configuration, :scheduler, 1 + assert_processes configuration, :scheduler, 0 end # No processes diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index f164bc1b..b2572b18 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -21,7 +21,7 @@ class SchedulerTest < ActiveSupport::TestCase SolidQueue::RecurringTask.create( key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -40,7 +40,7 @@ class SchedulerTest < ActiveSupport::TestCase recurring_tasks = { static_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } - scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks, dynamic_tasks: true).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -78,7 +78,7 @@ class SchedulerTest < ActiveSupport::TestCase key: "dynamic_enqueue_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) wait_while_with_timeout(3.seconds) { SolidQueue::Job.count < 1 } @@ -92,7 +92,7 @@ class SchedulerTest < ActiveSupport::TestCase end test "updates metadata after adding dynamic task post-start" do - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -124,7 +124,7 @@ class SchedulerTest < ActiveSupport::TestCase arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) From f7e39b5911cfe965a4d3b010138142763858fd69 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 14:54:52 +0100 Subject: [PATCH 22/27] Read static from options in from_configuration and rename to dynamic_tasks_enabled Have from_configuration read static from options instead of hardcoding it. Callers that create static tasks (YAML config) pass static: true via reverse_merge. schedule_recurring_task passes static: false directly. Rename the config key from dynamic_tasks to dynamic_tasks_enabled for clarity. Co-Authored-By: Claude Opus 4.6 --- README.md | 2 +- app/models/solid_queue/recurring_task.rb | 4 ++-- lib/solid_queue.rb | 5 +---- lib/solid_queue/configuration.rb | 6 +++--- lib/solid_queue/scheduler.rb | 4 ++-- lib/solid_queue/scheduler/recurring_schedule.rb | 6 +++--- test/unit/scheduler_test.rb | 10 +++++----- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ecb67501..f4f28ba7 100644 --- a/README.md +++ b/README.md @@ -781,7 +781,7 @@ SolidQueue.unschedule_recurring_task("my_dynamic_task") Only dynamic tasks can be unscheduled at runtime. Attempting to unschedule a static task (defined in `config/recurring.yml`) will raise an `ActiveRecord::RecordNotFound` error. -Tasks scheduled like this persist between Solid Queue's restarts and won't stop running until you manually unschedule them. +Tasks scheduled like this persist between Solid Queue's restarts and won't stop running until you manually unschedule them. ## Inspiration diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index d073a277..79fbce8d 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -20,7 +20,7 @@ class RecurringTask < Record class << self def wrap(args) - args.is_a?(self) ? args : from_configuration(args.first, **args.second) + args.is_a?(self) ? args : from_configuration(args.first, **args.second.reverse_merge(static: true)) end def from_configuration(key, **options) @@ -33,7 +33,7 @@ def from_configuration(key, **options) queue_name: options[:queue].presence, priority: options[:priority].presence, description: options[:description], - static: true + static: !!options[:static] end def create_or_update_all(tasks) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index af4b5a84..b46c1116 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,10 +44,7 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor def schedule_recurring_task(key, **options) - RecurringTask.from_configuration(key, **options.except(:static)).tap do |task| - task.static = false - task.save! - end + RecurringTask.from_configuration(key, **options, static: false).tap(&:save!) end def unschedule_recurring_task(key) diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index d0a46265..c6dfbe77 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -30,7 +30,7 @@ def instantiate SCHEDULER_DEFAULTS = { polling_interval: 5, - dynamic_tasks: false + dynamic_tasks_enabled: false } DEFAULT_CONFIG_FILE_PATH = "config/queue.yml" @@ -166,12 +166,12 @@ def scheduler_options end def dynamic_tasks_enabled? - scheduler_options.fetch(:dynamic_tasks, SCHEDULER_DEFAULTS[:dynamic_tasks]) + scheduler_options.fetch(:dynamic_tasks_enabled, SCHEDULER_DEFAULTS[:dynamic_tasks_enabled]) end def recurring_tasks @recurring_tasks ||= recurring_tasks_config.map do |id, options| - RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) + RecurringTask.from_configuration(id, **options.reverse_merge(static: true)) if options&.has_key?(:schedule) end.compact end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index c224f2b8..d9ea5c92 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -15,9 +15,9 @@ class Scheduler < Processes::Base def initialize(recurring_tasks:, **options) options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS) - @dynamic_tasks = options[:dynamic_tasks] + @dynamic_tasks = options[:dynamic_tasks_enabled] @polling_interval = options[:polling_interval] - @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks: @dynamic_tasks) + @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks_enabled: @dynamic_tasks) super(**options) end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 8b558271..6db997ca 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -6,11 +6,11 @@ class Scheduler::RecurringSchedule attr_reader :scheduled_tasks - def initialize(tasks, dynamic_tasks: false) - @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) + def initialize(static_tasks, dynamic_tasks_enabled: false) + @static_tasks = Array(static_tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @scheduled_tasks = Concurrent::Hash.new @changes = Concurrent::Hash.new - @dynamic_tasks_enabled = dynamic_tasks + @dynamic_tasks_enabled = dynamic_tasks_enabled end def configured_tasks diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index b2572b18..56aaeb2c 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -21,7 +21,7 @@ class SchedulerTest < ActiveSupport::TestCase SolidQueue::RecurringTask.create( key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks_enabled: true).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -40,7 +40,7 @@ class SchedulerTest < ActiveSupport::TestCase recurring_tasks = { static_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } - scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks, dynamic_tasks: true).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks, dynamic_tasks_enabled: true).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -78,7 +78,7 @@ class SchedulerTest < ActiveSupport::TestCase key: "dynamic_enqueue_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks_enabled: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) wait_while_with_timeout(3.seconds) { SolidQueue::Job.count < 1 } @@ -92,7 +92,7 @@ class SchedulerTest < ActiveSupport::TestCase end test "updates metadata after adding dynamic task post-start" do - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks_enabled: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -124,7 +124,7 @@ class SchedulerTest < ActiveSupport::TestCase arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks: true, polling_interval: 0.1).tap(&:start) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, dynamic_tasks_enabled: true, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) From b012535ae46acba69b74be1a248a43d598e2f6d7 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 15:29:18 +0100 Subject: [PATCH 23/27] Refactor scheduler loop and add missing tests Keep the longer sleep interval when dynamic tasks are disabled. Add tests for `dynamic_tasks_enabled` opt-in in configuration and for verifying dynamic tasks are ignored when not enabled. --- app/models/solid_queue/recurring_task.rb | 4 +- lib/solid_queue/configuration.rb | 6 +-- lib/solid_queue/scheduler.rb | 52 ++++++++++++------------ test/unit/configuration_test.rb | 18 ++++++++ test/unit/scheduler_test.rb | 16 ++++++++ 5 files changed, 66 insertions(+), 30 deletions(-) diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index 79fbce8d..7d719d6b 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -20,7 +20,7 @@ class RecurringTask < Record class << self def wrap(args) - args.is_a?(self) ? args : from_configuration(args.first, **args.second.reverse_merge(static: true)) + args.is_a?(self) ? args : from_configuration(args.first, **args.second) end def from_configuration(key, **options) @@ -33,7 +33,7 @@ def from_configuration(key, **options) queue_name: options[:queue].presence, priority: options[:priority].presence, description: options[:description], - static: !!options[:static] + static: options.fetch(:static, true) end def create_or_update_all(tasks) diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index c6dfbe77..3971ef82 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -144,8 +144,8 @@ def dispatchers def schedulers return [] if skip_recurring_tasks? - if recurring_tasks.any? || dynamic_tasks_enabled? - [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] + if recurring_tasks.any? || dynamic_recurring_tasks_enabled? + [ Process.new(:scheduler, { recurring_tasks: recurring_tasks, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] else [] end @@ -165,7 +165,7 @@ def scheduler_options @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys end - def dynamic_tasks_enabled? + def dynamic_recurring_tasks_enabled? scheduler_options.fetch(:dynamic_tasks_enabled, SCHEDULER_DEFAULTS[:dynamic_tasks_enabled]) end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index d9ea5c92..a7f7e085 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -15,9 +15,9 @@ class Scheduler < Processes::Base def initialize(recurring_tasks:, **options) options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS) - @dynamic_tasks = options[:dynamic_tasks_enabled] + @dynamic_tasks_enabled = options[:dynamic_tasks_enabled] @polling_interval = options[:polling_interval] - @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks_enabled: @dynamic_tasks) + @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks_enabled: @dynamic_tasks_enabled) super(**options) end @@ -27,35 +27,20 @@ def metadata end private - attr_reader :dynamic_tasks - def run - if dynamic_tasks - poll_for_dynamic_tasks - else - loop do - break if shutting_down? - interruptible_sleep(polling_interval) - end - end - ensure - SolidQueue.instrument(:shutdown_process, process: self) do - run_callbacks(:shutdown) { shutdown } - end - end + STATIC_SLEEP_INTERVAL = 60 - def poll_for_dynamic_tasks + def run loop do break if shutting_down? - recurring_schedule.reload! - - if recurring_schedule.changed? - refresh_registered_process - recurring_schedule.clear_changes - end + reload_schedule if dynamic_tasks_enabled? - interruptible_sleep(polling_interval) + interruptible_sleep(sleep_interval) + end + ensure + SolidQueue.instrument(:shutdown_process, process: self) do + run_callbacks(:shutdown) { shutdown } end end @@ -67,10 +52,27 @@ def unschedule_recurring_tasks recurring_schedule.unschedule_tasks end + def reload_schedule + recurring_schedule.reload! + + if recurring_schedule.changed? + refresh_registered_process + recurring_schedule.clear_changes + end + end + + def dynamic_tasks_enabled? + @dynamic_tasks_enabled + end + def all_work_completed? recurring_schedule.empty? end + def sleep_interval + dynamic_tasks_enabled? ? polling_interval : STATIC_SLEEP_INTERVAL + end + def set_procline procline "scheduling #{recurring_schedule.task_keys.join(",")}" end diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 11c2a5ff..34f69658 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -87,6 +87,24 @@ class ConfigurationTest < ActiveSupport::TestCase assert_has_recurring_task scheduler, key: "periodic_store_result", class_name: "StoreResultJob", schedule: "every second" end + test "scheduler starts with dynamic_tasks_enabled even without static tasks" do + configuration = SolidQueue::Configuration.new( + recurring_schedule_file: config_file_path(:empty_configuration), + scheduler: { dynamic_tasks_enabled: true } + ) + + assert_processes configuration, :scheduler, 1, dynamic_tasks_enabled: true + end + + test "no scheduler without static tasks or dynamic_tasks_enabled" do + configuration = SolidQueue::Configuration.new( + recurring_schedule_file: config_file_path(:empty_configuration), + scheduler: { dynamic_tasks_enabled: false } + ) + + assert_processes configuration, :scheduler, 0 + end + test "no recurring tasks configuration when explicitly excluded" do configuration = SolidQueue::Configuration.new(dispatchers: [ { polling_interval: 0.1 } ], skip_recurring: true) assert_processes configuration, :dispatcher, 1, polling_interval: 0.1, recurring_tasks: nil diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 56aaeb2c..e914a23c 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -52,6 +52,22 @@ class SchedulerTest < ActiveSupport::TestCase scheduler.stop end + test "dynamic tasks in DB are ignored when dynamic_tasks_enabled is false" do + SolidQueue::RecurringTask.create!( + key: "ignored_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + + recurring_tasks = { static_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } + scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_metadata process, recurring_schedule: [ "static_task" ] + ensure + scheduler.stop + end + test "run more than one instance of the scheduler with recurring tasks" do recurring_tasks = { example_task: { class: "AddToBufferJob", schedule: "every second", args: 42 } } schedulers = 2.times.collect do From 3d36ca4acb327cc51dd5da5c57f4b2d2951c7c8a Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 16:03:56 +0100 Subject: [PATCH 24/27] Simplify recurring schedule with respect to dynamic task reloading We don't need to track dynamic task changes, we can just reload the metadata in every case. If it hasn't changed, Rails won't issue any new update to the process record. Also, we can just use `dynamic_tasks` everywhere, as an empty AR relation if dynamic tasks are disabled, avoiding extra queries but keeping the code simpler. --- lib/solid_queue/processes/registrable.rb | 4 +- lib/solid_queue/scheduler.rb | 12 ++-- .../scheduler/recurring_schedule.rb | 70 +++++++------------ 3 files changed, 30 insertions(+), 56 deletions(-) diff --git a/lib/solid_queue/processes/registrable.rb b/lib/solid_queue/processes/registrable.rb index 94e9ed77..35b4e01b 100644 --- a/lib/solid_queue/processes/registrable.rb +++ b/lib/solid_queue/processes/registrable.rb @@ -60,8 +60,8 @@ def heartbeat wake_up end - def refresh_registered_process - wrap_in_app_executor { process&.update_columns(metadata: metadata.compact) } + def reload_metadata + wrap_in_app_executor { process&.update(metadata: metadata.compact) } end end end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index a7f7e085..789a6c13 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -34,7 +34,7 @@ def run loop do break if shutting_down? - reload_schedule if dynamic_tasks_enabled? + reload_dynamic_schedule if dynamic_tasks_enabled? interruptible_sleep(sleep_interval) end @@ -52,13 +52,9 @@ def unschedule_recurring_tasks recurring_schedule.unschedule_tasks end - def reload_schedule - recurring_schedule.reload! - - if recurring_schedule.changed? - refresh_registered_process - recurring_schedule.clear_changes - end + def reload_dynamic_schedule + recurring_schedule.reload_dynamic_tasks + reload_metadata end def dynamic_tasks_enabled? diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 6db997ca..ee6a174c 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -7,26 +7,18 @@ class Scheduler::RecurringSchedule attr_reader :scheduled_tasks def initialize(static_tasks, dynamic_tasks_enabled: false) - @static_tasks = Array(static_tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) - @scheduled_tasks = Concurrent::Hash.new - @changes = Concurrent::Hash.new + @static_tasks = Array(static_tasks).map { |task| RecurringTask.wrap(task) }.select(&:valid?) @dynamic_tasks_enabled = dynamic_tasks_enabled + + @scheduled_tasks = Concurrent::Hash.new end def configured_tasks - if dynamic_tasks_enabled? - static_tasks + dynamic_tasks.to_a - else - static_tasks - end + static_tasks + dynamic_tasks end def empty? - if dynamic_tasks_enabled? - scheduled_tasks.empty? && dynamic_tasks.none? - else - scheduled_tasks.empty? - end + scheduled_tasks.empty? && dynamic_tasks.empty? end def schedule_tasks @@ -50,69 +42,55 @@ def unschedule_tasks end def task_keys - if dynamic_tasks_enabled? - static_task_keys + dynamic_tasks.pluck(:key) - else - static_task_keys - end + static_task_keys + dynamic_task_keys end - def reload! + def reload_dynamic_tasks wrap_in_app_executor do - { added_tasks: schedule_new_dynamic_tasks, - removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| - if values.any? - @changes[key] = values - else - @changes.delete(key) - end - end + schedule_created_dynamic_tasks + unschedule_deleted_dynamic_tasks end end - def changed? - @changes.any? - end - - def clear_changes - @changes.clear - end - private attr_reader :static_tasks - def dynamic_tasks_enabled? - @dynamic_tasks_enabled + def static_task_keys + static_tasks.map(&:key) end def dynamic_tasks - SolidQueue::RecurringTask.dynamic + dynamic_tasks_enabled? ? RecurringTask.dynamic : RecurringTask.none end - def static_task_keys - static_tasks.map(&:key) + def dynamic_task_keys + dynamic_tasks.pluck(:key) + end + + def dynamic_tasks_enabled? + @dynamic_tasks_enabled end - def schedule_new_dynamic_tasks + def schedule_created_dynamic_tasks dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| schedule_task(task) end end - def unschedule_old_dynamic_tasks - (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| + def unschedule_deleted_dynamic_tasks + (scheduled_tasks.keys - RecurringTask.pluck(:key)).each do |key| scheduled_tasks[key].cancel scheduled_tasks.delete(key) end end def persist_static_tasks - SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all - SolidQueue::RecurringTask.create_or_update_all static_tasks + RecurringTask.static.where.not(key: static_task_keys).delete_all + RecurringTask.create_or_update_all static_tasks end def reload_static_tasks - @static_tasks = SolidQueue::RecurringTask.static.where(key: static_task_keys).to_a + @static_tasks = RecurringTask.static.where(key: static_task_keys).to_a end def schedule(task) From b4f7c99ac3a1c2436e2accbad737eedeba94dbc3 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 16:13:02 +0100 Subject: [PATCH 25/27] Clean a bit top-level SolidQueue methods to manage dynamic tasks --- app/models/solid_queue/recurring_task.rb | 8 ++++++++ lib/solid_queue.rb | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index 7d719d6b..943f01b8 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -36,6 +36,14 @@ def from_configuration(key, **options) static: options.fetch(:static, true) end + def create_dynamic_task(key, **options) + from_configuration(key, **options.reverse_merge(static: false)).save! + end + + def delete_dynamic_task(key) + RecurringTask.dynamic.find_by!(key: key).destroy + end + def create_or_update_all(tasks) if supports_insert_conflict_target? # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index b46c1116..bd2269e5 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,11 +44,11 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor def schedule_recurring_task(key, **options) - RecurringTask.from_configuration(key, **options, static: false).tap(&:save!) + RecurringTask.create_dynamic_task(key, **options) end def unschedule_recurring_task(key) - RecurringTask.dynamic.find_by!(key:).destroy + RecurringTask.delete_dynamic_task(key) end [ Dispatcher, Scheduler, Worker ].each do |process| From 7ded7101b97b41552a241ba3b3b562b961250a96 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 16:44:48 +0100 Subject: [PATCH 26/27] Avoid extra queries for dynamic tasks for process metadata and procline We need the dynamic task keys there and were doing a new query every time. We only need to do a query when explicitly reloading them. --- lib/solid_queue/scheduler.rb | 2 +- .../scheduler/recurring_schedule.rb | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 789a6c13..6022b338 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -53,7 +53,7 @@ def unschedule_recurring_tasks end def reload_dynamic_schedule - recurring_schedule.reload_dynamic_tasks + recurring_schedule.reschedule_dynamic_tasks reload_metadata end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index ee6a174c..a1e2409e 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -25,6 +25,7 @@ def schedule_tasks wrap_in_app_executor do persist_static_tasks reload_static_tasks + reload_dynamic_tasks end configured_tasks.each do |task| @@ -42,11 +43,12 @@ def unschedule_tasks end def task_keys - static_task_keys + dynamic_task_keys + configured_tasks.map(&:key) end - def reload_dynamic_tasks + def reschedule_dynamic_tasks wrap_in_app_executor do + reload_dynamic_tasks schedule_created_dynamic_tasks unschedule_deleted_dynamic_tasks end @@ -60,11 +62,7 @@ def static_task_keys end def dynamic_tasks - dynamic_tasks_enabled? ? RecurringTask.dynamic : RecurringTask.none - end - - def dynamic_task_keys - dynamic_tasks.pluck(:key) + @dynamic_tasks ||= load_dynamic_tasks end def dynamic_tasks_enabled? @@ -72,7 +70,7 @@ def dynamic_tasks_enabled? end def schedule_created_dynamic_tasks - dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| + RecurringTask.dynamic.where.not(key: scheduled_tasks.keys).each do |task| schedule_task(task) end end @@ -93,6 +91,14 @@ def reload_static_tasks @static_tasks = RecurringTask.static.where(key: static_task_keys).to_a end + def reload_dynamic_tasks + @dynamic_tasks = load_dynamic_tasks + end + + def load_dynamic_tasks + dynamic_tasks_enabled? ? RecurringTask.dynamic.to_a : [] + end + def schedule(task) scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at| thread_schedule.schedule_task(thread_task) From 3b490add2cd4de885269ff31682ec1519758ffb9 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Wed, 25 Feb 2026 18:48:26 +0100 Subject: [PATCH 27/27] Merge right value of `static` correctly We want to set that value overriding whatever we get in options, so it should be merged into options, not the other way around. Co-Authored-By: Vladyslav Davydenko --- app/models/solid_queue/recurring_task.rb | 2 +- lib/solid_queue/configuration.rb | 2 +- .../models/solid_queue/recurring_task_test.rb | 50 +++++++++++++++++++ test/solid_queue_test.rb | 50 ------------------- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index 943f01b8..9bb634e6 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -37,7 +37,7 @@ def from_configuration(key, **options) end def create_dynamic_task(key, **options) - from_configuration(key, **options.reverse_merge(static: false)).save! + from_configuration(key, **options.merge(static: false)).save! end def delete_dynamic_task(key) diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 3971ef82..e63a000c 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -171,7 +171,7 @@ def dynamic_recurring_tasks_enabled? def recurring_tasks @recurring_tasks ||= recurring_tasks_config.map do |id, options| - RecurringTask.from_configuration(id, **options.reverse_merge(static: true)) if options&.has_key?(:schedule) + RecurringTask.from_configuration(id, **options.merge(static: true)) if options&.has_key?(:schedule) end.compact end diff --git a/test/models/solid_queue/recurring_task_test.rb b/test/models/solid_queue/recurring_task_test.rb index 52312f12..dba9d6b9 100644 --- a/test/models/solid_queue/recurring_task_test.rb +++ b/test/models/solid_queue/recurring_task_test.rb @@ -210,6 +210,56 @@ def perform assert_equal "SolidQueue::RecurringJob", SolidQueue::Job.last.class_name end + test "schedule recurring tasks dynamically" do + SolidQueue::RecurringTask.create_dynamic_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue::RecurringTask.create_dynamic_task("test 2", command: "puts 2", schedule: "every minute", static: true) + + assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) + assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) + end + + test "schedule recurring tasks dynamically with class and args (same keys as YAML config)" do + SolidQueue::RecurringTask.create_dynamic_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + + task = SolidQueue::RecurringTask.find_by!(key: "test 3") + assert_equal "AddToBufferJob", task.class_name + assert_equal [ 42 ], task.arguments + assert_not task.static + end + + test "unschedule recurring tasks dynamically" do + dynamic_task = SolidQueue::RecurringTask.create!( + key: "dynamic", command: "puts 'd'", schedule: "every day", static: false + ) + + static_task = SolidQueue::RecurringTask.create!( + key: "static", command: "puts 's'", schedule: "every week", static: true + ) + + SolidQueue::RecurringTask.delete_dynamic_task(dynamic_task.key) + + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue::RecurringTask.delete_dynamic_task(static_task.key) + end + + assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) + assert SolidQueue::RecurringTask.exists?(key: "static", static: true) + end + + test "scheduling dynamic recurring task with duplicate key raises error" do + SolidQueue::RecurringTask.create_dynamic_task("duplicate_test", command: "puts 1", schedule: "every hour") + + assert_raises(ActiveRecord::RecordNotUnique) do + SolidQueue::RecurringTask.create_dynamic_task("duplicate_test", command: "puts 2", schedule: "every minute") + end + end + + test "unscheduling dynamic recurring task with nonexistent key raises RecordNotFound" do + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue::RecurringTask.delete_dynamic_task("nonexistent_key") + end + end + private def enqueue_and_assert_performed_with_result(task, result) assert_difference [ -> { SolidQueue::Job.count }, -> { SolidQueue::ReadyExecution.count } ], +1 do diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 768b9c3c..d6d61b57 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -4,54 +4,4 @@ class SolidQueueTest < ActiveSupport::TestCase test "it has a version number" do assert SolidQueue::VERSION end - - test "schedules recurring tasks" do - SolidQueue.schedule_recurring_task("test 1", command: "puts 1", schedule: "every hour") - SolidQueue.schedule_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) - - assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) - assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) - end - - test "schedules recurring tasks with class and args (same keys as YAML config)" do - SolidQueue.schedule_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") - - task = SolidQueue::RecurringTask.find_by!(key: "test 3") - assert_equal "AddToBufferJob", task.class_name - assert_equal [ 42 ], task.arguments - assert_equal false, task.static - end - - test "unschedules recurring tasks" do - dynamic_task = SolidQueue::RecurringTask.create!( - key: "dynamic", command: "puts 'd'", schedule: "every day", static: false - ) - - static_task = SolidQueue::RecurringTask.create!( - key: "static", command: "puts 's'", schedule: "every week", static: true - ) - - SolidQueue.unschedule_recurring_task(dynamic_task.key) - - assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.unschedule_recurring_task(static_task.key) - end - - assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) - assert SolidQueue::RecurringTask.exists?(key: "static", static: true) - end - - test "schedule_recurring_task with duplicate key raises error" do - SolidQueue.schedule_recurring_task("duplicate_test", command: "puts 1", schedule: "every hour") - - assert_raises(ActiveRecord::RecordNotUnique) do - SolidQueue.schedule_recurring_task("duplicate_test", command: "puts 2", schedule: "every minute") - end - end - - test "unschedule_recurring_task with nonexistent key raises RecordNotFound" do - assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.unschedule_recurring_task("nonexistent_key") - end - end end