hosting.queues.inc

This file defines an API for defining new queues.

Functions

Namesort descending Description
hosting_get_queues Retrieve a list of queues that need to be dispatched
hosting_queues_cron_cmd Generate a crontab entry.
hosting_queues_cron_removal Remove Aegir lines from a crontab.
hosting_run_queue Run a queue specified by hook_hosting_queues()
_hosting_dispatch_cmd
_hosting_queues_clean_output
_hosting_queue_next_run Calculate the time at which the task queue will run next.

File

hosting.queues.inc
View source
  1. <?php
  2. /**
  3. * @file
  4. * This file defines an API for defining new queues.
  5. */
  6. /**
  7. * @defgroup backend-frontend-IPC Frontend/Backend Inter-process Communication
  8. * @{
  9. * Aegir is based on a very important distinction between the
  10. * "frontend" and the "backend". The frontend is the Drupal website
  11. * that creates the user interface that people manipulate. The backend
  12. * is (usually) a set a drush commands that do the actual operations
  13. * on the backend objects (servers, platforms, sites).
  14. *
  15. * The frontend and backend run as different users: the frontend runs
  16. * under the webserver user, while the backend run as a separate UNIX
  17. * user, which we call the backend sandbox. This distinction is
  18. * important as it brings an extra layer of security. It also allows
  19. * for commands to be ran on different servers more easily.
  20. *
  21. * The way the frontend communicates to the backend is through the
  22. * use of "queues", that are actually defined in the frontend
  23. * (hosting). The frontend also define drush commands that are ran
  24. * through a cronjob and therefore provide the glue to the backend.
  25. *
  26. * The point of entry is through the "dispatcher" (the
  27. * hosting-dispatch command), which runs the different queues
  28. * appropriately. There are different types of queues (see
  29. * hook_hosting_queues()), but the most important one is the "task"
  30. * queue, which runs the basic tasks like install and backup.
  31. *
  32. * When tasks are ran, a provision command is called with the same
  33. * name of the task type defined in the frontend (see
  34. * drush_hosting_task()). If we are running a special task like
  35. * install, verify or import, data from the task is actually saved in
  36. * the backend, through the backend provision-save command. The data
  37. * saved is from the associative array "context_options", defined from
  38. * the hook see hook_hosting_TASK_OBJECT_context_options(). A good
  39. * example of such a variable is the site name or platform it is on,
  40. * which are obviously saved in the backend.
  41. *
  42. * There is also an "options" array that are just one-time parameters
  43. * that are not stored in the backend context. A good example of that
  44. * is the site email address, that is just sent during the install or
  45. * password resets task but not saved.
  46. */
  47. /**
  48. * Retrieve a list of queues that need to be dispatched
  49. *
  50. * Generate a list of queues, and the frequency / amount of items
  51. * that need to be processed for each of them.
  52. */
  53. function hosting_get_queues($refresh = FALSE) {
  54. static $cache = NULL;
  55. if (is_null($cache) || $refresh) {
  56. $cache = array();
  57. $defaults = array(
  58. 'type' => 'serial',
  59. 'max_threads' => 6,
  60. 'threshold' => '100',
  61. 'min_threads' => 1,
  62. 'timeout' => strtotime("10 minutes", 0),
  63. 'frequency' => strtotime("5 minutes", 0),
  64. 'items' => 5,
  65. 'enabled' => TRUE,
  66. 'singular' => t('item'),
  67. 'plural' => t('items'),
  68. );
  69. $queues = module_invoke_all("hosting_queues");
  70. foreach ($queues as $key => $queue) {
  71. $queue = array_merge($defaults, $queue);
  72. // Configurable settings.
  73. $configured = array(
  74. 'frequency' => variable_get('hosting_queue_' . $key . '_frequency', $queue['frequency']),
  75. 'items' => variable_get('hosting_queue_' . $key . '_items', $queue['items']),
  76. 'enabled' => variable_get('hosting_queue_' . $key . '_enabled', $queue['enabled']),
  77. 'last_run' => variable_get('hosting_queue_' . $key . '_last_run', FALSE),
  78. 'interval' => variable_get('hosting_queue_' . $key . '_interval', FALSE),
  79. );
  80. $queue = array_merge($queue, $configured);
  81. if ($queue['type'] == 'batch') {
  82. $threads = $queue['total_items'] / $queue['threshold'];
  83. if ($threads <= $queue['min_threads']) {
  84. $threads = $queue['min_threads'];
  85. }
  86. elseif ($thread > $queue['max_threads']) {
  87. $threads = $queue['max_threads'];
  88. }
  89. $queue['calc_threads'] = $threads;
  90. $queue['calc_frequency'] = ceil($queue['frequency'] / $threads);
  91. $queue['calc_items'] = ceil($queue['total_items'] / $threads);
  92. }
  93. else {
  94. $queue['calc_frequency'] = $queue['frequency'];
  95. $queue['calc_items'] = $queue['items'];
  96. }
  97. $queue['last'] = variable_get('hosting_queue_' . $key . '_last_run', 0);
  98. $queues[$key] = $queue;
  99. }
  100. $cache = $queues;
  101. }
  102. return $cache;
  103. }
  104. /**
  105. * Run a queue specified by hook_hosting_queues()
  106. *
  107. * Run an instance of a queue processor. This function contains all the book keeping
  108. * functionality needed to ensure that the queues are running as scheduled.
  109. */
  110. function hosting_run_queue() {
  111. $cmd = drush_get_command();
  112. $queue = $cmd['queue'];
  113. $count = drush_get_option(array('i', 'items'), 5); // process a default of 5 items at a time.
  114. $semaphore = "hosting_queue_{$queue}_running";
  115. variable_set('hosting_queue_' . $queue . '_last_run', $t = REQUEST_TIME);
  116. drush_log(dt('Acquiring lock on @queue queue.', array('@queue' => $queue)));
  117. $lock_wait = drush_get_option('lock-wait', HOSTING_QUEUE_DEFAULT_LOCK_WAIT);
  118. if (!lock_wait($semaphore, $lock_wait) || drush_get_option('force', FALSE)) {
  119. if (lock_acquire($semaphore, HOSTING_QUEUE_LOCK_TIMEOUT)) {
  120. drush_log(dt('Acquired lock on @queue queue.', array('@queue' => $queue)));
  121. }
  122. }
  123. elseif (drush_get_option('force', FALSE)) {
  124. drush_log(dt('Bypassing lock on @queue queue.', array('@queue' => $queue)), 'warning');
  125. }
  126. else {
  127. drush_die(dt('Cannot acquire lock on @queue queue.', array('@queue' => $queue)));
  128. }
  129. $func = "hosting_" . $queue . "_queue";
  130. if (function_exists($func)) {
  131. $func($count);
  132. }
  133. drush_log(dt('Releasing @queue lock.', array('@queue' => $queue)));
  134. lock_release($semaphore);
  135. }
  136. /**
  137. * Calculate the time at which the task queue will run next.
  138. *
  139. * There are two possibilities here - the task was never run, in which
  140. * case we compute the next time it was run.
  141. *
  142. * The second case is the case where the task wasn't run in the last
  143. * queue run even though it was scheduled, so we compute the next
  144. * iteration.
  145. *
  146. * @param $queue mixed
  147. * the queue name or a queue array as returned by
  148. * hosting_get_queues()
  149. *
  150. * @return integer
  151. * the date at which the queue will run as a number of seconds since
  152. * the UNIX epoch
  153. */
  154. function _hosting_queue_next_run($queue) {
  155. if (!is_array($queue)) {
  156. $queues = hosting_get_queues();
  157. $queue = $queues[$queue];
  158. }
  159. $freq = $queue['frequency'];
  160. $last = $queue['last_run'];
  161. $now = REQUEST_TIME;
  162. return max($last + $freq,
  163. ( $now - ( ($now - $last) % $freq ) ) + $freq );
  164. }
  165. function _hosting_queues_clean_output($return) {
  166. return filter_xss($return, array());
  167. }
  168. function _hosting_dispatch_cmd() {
  169. $cmd = sprintf("/usr/bin/env php %s %s hosting-dispatch ", DRUSH_COMMAND, escapeshellarg(d()->name));
  170. return $cmd;
  171. }
  172. /**
  173. * Generate a crontab entry.
  174. *
  175. * @return string
  176. * The lines to be added.
  177. *
  178. * @see hosting_queues_cron_removal()
  179. */
  180. function hosting_queues_cron_cmd() {
  181. $command = _hosting_dispatch_cmd();
  182. $return = <<<END
  183. SHELL=/bin/sh
  184. PATH=$_SERVER[PATH]
  185. * * * * * $command
  186. END;
  187. return $return;
  188. }
  189. /**
  190. * Remove Aegir lines from a crontab.
  191. *
  192. * @param array $crontab
  193. * Multi line array of the current crontab entry.
  194. *
  195. * @return array
  196. * The left over non-Aegir lines.
  197. *
  198. * @see hosting_queues_cron_cmd()
  199. */
  200. function hosting_queues_cron_removal($crontab) {
  201. foreach ($crontab as $k => $cron) {
  202. if (preg_match('|SHELL=/bin/sh|', $cron)) {
  203. unset($crontab[$k]);
  204. }
  205. if (preg_match("|PATH=$_SERVER[PATH]|", $cron)) {
  206. unset($crontab[$k]);
  207. }
  208. if (preg_match('|hosting-dispatch|', $cron)) {
  209. unset($crontab[$k]);
  210. }
  211. }
  212. return $crontab;
  213. }
  214. /**
  215. * @} End of "defgroup backend-frontend-IPC".
  216. */