Page MenuHomestyx hydra
Files F127087

No OneTemporary
Actions

View Options

diff --git a/resources/sql/autopatches/20181024.drydock.01.commandprops.sql b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql
new file mode 100644
index 0000000000..e808146b02
--- /dev/null
+++ b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_drydock.drydock_command
+ ADD properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT};
diff --git a/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql
new file mode 100644
index 0000000000..2c336dc40e
--- /dev/null
+++ b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql
@@ -0,0 +1,2 @@
+UPDATE {$NAMESPACE}_drydock.drydock_command
+ SET properties = '{}' WHERE properties = '';
diff --git a/src/applications/drydock/storage/DrydockCommand.php b/src/applications/drydock/storage/DrydockCommand.php
index e7d003bdd6..0f7f253217 100644
--- a/src/applications/drydock/storage/DrydockCommand.php
+++ b/src/applications/drydock/storage/DrydockCommand.php
@@ -1,70 +1,82 @@
<?php
final class DrydockCommand
extends DrydockDAO
implements PhabricatorPolicyInterface {
const COMMAND_RELEASE = 'release';
const COMMAND_RECLAIM = 'reclaim';
protected $authorPHID;
protected $targetPHID;
protected $command;
protected $isConsumed;
+ protected $properties = array();
private $commandTarget = self::ATTACHABLE;
public static function initializeNewCommand(PhabricatorUser $author) {
return id(new DrydockCommand())
->setAuthorPHID($author->getPHID())
->setIsConsumed(0);
}
protected function getConfiguration() {
return array(
+ self::CONFIG_SERIALIZATION => array(
+ 'properties' => self::SERIALIZATION_JSON,
+ ),
self::CONFIG_COLUMN_SCHEMA => array(
'command' => 'text32',
'isConsumed' => 'bool',
),
self::CONFIG_KEY_SCHEMA => array(
'key_target' => array(
'columns' => array('targetPHID', 'isConsumed'),
),
),
) + parent::getConfiguration();
}
public function attachCommandTarget($target) {
$this->commandTarget = $target;
return $this;
}
public function getCommandTarget() {
return $this->assertAttached($this->commandTarget);
}
+ public function setProperty($key, $value) {
+ $this->properties[$key] = $value;
+ return $this;
+ }
+
+ public function getProperty($key, $default = null) {
+ return idx($this->properties, $key, $default);
+ }
/* -( PhabricatorPolicyInterface )----------------------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
);
}
public function getPolicy($capability) {
return $this->getCommandTarget()->getPolicy($capability);
}
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
return $this->getCommandTarget()->hasAutomaticCapability(
$capability,
$viewer);
}
public function describeAutomaticCapability($capability) {
return pht('Drydock commands have the same policies as their targets.');
}
}
diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
index baa80f90d8..817fbaddee 100644
--- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
+++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
@@ -1,301 +1,308 @@
<?php
/**
* @task update Updating Resources
* @task command Processing Commands
* @task activate Activating Resources
* @task release Releasing Resources
* @task break Breaking Resources
* @task destroy Destroying Resources
*/
final class DrydockResourceUpdateWorker extends DrydockWorker {
protected function doWork() {
$resource_phid = $this->getTaskDataValue('resourcePHID');
$hash = PhabricatorHash::digestForIndex($resource_phid);
$lock_key = 'drydock.resource:'.$hash;
$lock = PhabricatorGlobalLock::newLock($lock_key)
->lock(1);
try {
$resource = $this->loadResource($resource_phid);
$this->handleUpdate($resource);
} catch (Exception $ex) {
$lock->unlock();
$this->flushDrydockTaskQueue();
throw $ex;
}
$lock->unlock();
}
/* -( Updating Resources )------------------------------------------------- */
/**
* Update a resource, handling exceptions thrown during the update.
*
* @param DrydockReosource Resource to update.
* @return void
* @task update
*/
private function handleUpdate(DrydockResource $resource) {
try {
$this->updateResource($resource);
} catch (Exception $ex) {
if ($this->isTemporaryException($ex)) {
$this->yieldResource($resource, $ex);
} else {
$this->breakResource($resource, $ex);
}
}
}
/**
* Update a resource.
*
* @param DrydockResource Resource to update.
* @return void
* @task update
*/
private function updateResource(DrydockResource $resource) {
$this->processResourceCommands($resource);
$resource_status = $resource->getStatus();
switch ($resource_status) {
case DrydockResourceStatus::STATUS_PENDING:
$this->activateResource($resource);
break;
case DrydockResourceStatus::STATUS_ACTIVE:
// Nothing to do.
break;
case DrydockResourceStatus::STATUS_RELEASED:
case DrydockResourceStatus::STATUS_BROKEN:
$this->destroyResource($resource);
break;
case DrydockResourceStatus::STATUS_DESTROYED:
// Nothing to do.
break;
}
$this->yieldIfExpiringResource($resource);
}
/**
* Convert a temporary exception into a yield.
*
* @param DrydockResource Resource to yield.
* @param Exception Temporary exception worker encountered.
* @task update
*/
private function yieldResource(DrydockResource $resource, Exception $ex) {
$duration = $this->getYieldDurationFromException($ex);
$resource->logEvent(
DrydockResourceActivationYieldLogType::LOGCONST,
array(
'duration' => $duration,
));
throw new PhabricatorWorkerYieldException($duration);
}
/* -( Processing Commands )------------------------------------------------ */
/**
* @task command
*/
private function processResourceCommands(DrydockResource $resource) {
if (!$resource->canReceiveCommands()) {
return;
}
$this->checkResourceExpiration($resource);
$commands = $this->loadCommands($resource->getPHID());
foreach ($commands as $command) {
if (!$resource->canReceiveCommands()) {
break;
}
$this->processResourceCommand($resource, $command);
$command
->setIsConsumed(true)
->save();
}
}
/**
* @task command
*/
private function processResourceCommand(
DrydockResource $resource,
DrydockCommand $command) {
switch ($command->getCommand()) {
case DrydockCommand::COMMAND_RELEASE:
$this->releaseResource($resource, null);
break;
case DrydockCommand::COMMAND_RECLAIM:
$reclaimer_phid = $command->getAuthorPHID();
$this->releaseResource($resource, $reclaimer_phid);
break;
}
+
+ // If the command specifies that other worker tasks should be awakened
+ // after it executes, awaken them now.
+ $awaken_ids = $command->getProperty('awakenTaskIDs');
+ if (is_array($awaken_ids) && $awaken_ids) {
+ PhabricatorWorker::awakenTaskIDs($awaken_ids);
+ }
}
/* -( Activating Resources )----------------------------------------------- */
/**
* @task activate
*/
private function activateResource(DrydockResource $resource) {
$blueprint = $resource->getBlueprint();
$blueprint->activateResource($resource);
$this->validateActivatedResource($blueprint, $resource);
}
/**
* @task activate
*/
private function validateActivatedResource(
DrydockBlueprint $blueprint,
DrydockResource $resource) {
if (!$resource->isActivatedResource()) {
throw new Exception(
pht(
'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
'must actually allocate the resource it returns.',
$blueprint->getBlueprintName(),
$blueprint->getClassName(),
'allocateResource()'));
}
}
/* -( Releasing Resources )------------------------------------------------ */
/**
* @task release
*/
private function releaseResource(
DrydockResource $resource,
$reclaimer_phid) {
if ($reclaimer_phid) {
if (!$this->canReclaimResource($resource)) {
return;
}
$resource->logEvent(
DrydockResourceReclaimLogType::LOGCONST,
array(
'reclaimerPHID' => $reclaimer_phid,
));
}
$viewer = $this->getViewer();
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
$resource
->setStatus(DrydockResourceStatus::STATUS_RELEASED)
->save();
$statuses = array(
DrydockLeaseStatus::STATUS_PENDING,
DrydockLeaseStatus::STATUS_ACQUIRED,
DrydockLeaseStatus::STATUS_ACTIVE,
);
$leases = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withResourcePHIDs(array($resource->getPHID()))
->withStatuses($statuses)
->execute();
foreach ($leases as $lease) {
$command = DrydockCommand::initializeNewCommand($viewer)
->setTargetPHID($lease->getPHID())
->setAuthorPHID($drydock_phid)
->setCommand(DrydockCommand::COMMAND_RELEASE)
->save();
$lease->scheduleUpdate();
}
$this->destroyResource($resource);
}
/* -( Breaking Resources )------------------------------------------------- */
/**
* @task break
*/
private function breakResource(DrydockResource $resource, Exception $ex) {
switch ($resource->getStatus()) {
case DrydockResourceStatus::STATUS_BROKEN:
case DrydockResourceStatus::STATUS_RELEASED:
case DrydockResourceStatus::STATUS_DESTROYED:
// If the resource was already broken, just throw a normal exception.
// This will retry the task eventually.
throw new PhutilProxyException(
pht(
'Unexpected failure while destroying resource ("%s").',
$resource->getPHID()),
$ex);
}
$resource
->setStatus(DrydockResourceStatus::STATUS_BROKEN)
->save();
$resource->scheduleUpdate();
$resource->logEvent(
DrydockResourceActivationFailureLogType::LOGCONST,
array(
'class' => get_class($ex),
'message' => $ex->getMessage(),
));
throw new PhabricatorWorkerPermanentFailureException(
pht(
'Permanent failure while activating resource ("%s"): %s',
$resource->getPHID(),
$ex->getMessage()));
}
/* -( Destroying Resources )----------------------------------------------- */
/**
* @task destroy
*/
private function destroyResource(DrydockResource $resource) {
$blueprint = $resource->getBlueprint();
$blueprint->destroyResource($resource);
DrydockSlotLock::releaseLocks($resource->getPHID());
$resource
->setStatus(DrydockResourceStatus::STATUS_DESTROYED)
->save();
}
}
diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php
index 443780680d..8a81da09b4 100644
--- a/src/applications/drydock/worker/DrydockWorker.php
+++ b/src/applications/drydock/worker/DrydockWorker.php
@@ -1,255 +1,260 @@
<?php
abstract class DrydockWorker extends PhabricatorWorker {
protected function getViewer() {
return PhabricatorUser::getOmnipotentUser();
}
protected function loadLease($lease_phid) {
$viewer = $this->getViewer();
$lease = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withPHIDs(array($lease_phid))
->executeOne();
if (!$lease) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such lease "%s"!', $lease_phid));
}
return $lease;
}
protected function loadResource($resource_phid) {
$viewer = $this->getViewer();
$resource = id(new DrydockResourceQuery())
->setViewer($viewer)
->withPHIDs(array($resource_phid))
->executeOne();
if (!$resource) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such resource "%s"!', $resource_phid));
}
return $resource;
}
protected function loadOperation($operation_phid) {
$viewer = $this->getViewer();
$operation = id(new DrydockRepositoryOperationQuery())
->setViewer($viewer)
->withPHIDs(array($operation_phid))
->executeOne();
if (!$operation) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such operation "%s"!', $operation_phid));
}
return $operation;
}
protected function loadCommands($target_phid) {
$viewer = $this->getViewer();
$commands = id(new DrydockCommandQuery())
->setViewer($viewer)
->withTargetPHIDs(array($target_phid))
->withConsumed(false)
->execute();
$commands = msort($commands, 'getID');
return $commands;
}
protected function checkLeaseExpiration(DrydockLease $lease) {
$this->checkObjectExpiration($lease);
}
protected function checkResourceExpiration(DrydockResource $resource) {
$this->checkObjectExpiration($resource);
}
private function checkObjectExpiration($object) {
// Check if the resource or lease has expired. If it has, we're going to
// send it a release command.
// This command is sent from within the update worker so it is handled
// immediately, but doing this generates a log and improves consistency.
$expires = $object->getUntil();
if (!$expires) {
return;
}
$now = PhabricatorTime::getNow();
if ($expires > $now) {
return;
}
$viewer = $this->getViewer();
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
$command = DrydockCommand::initializeNewCommand($viewer)
->setTargetPHID($object->getPHID())
->setAuthorPHID($drydock_phid)
->setCommand(DrydockCommand::COMMAND_RELEASE)
->save();
}
protected function yieldIfExpiringLease(DrydockLease $lease) {
if (!$lease->canReceiveCommands()) {
return;
}
$this->yieldIfExpiring($lease->getUntil());
}
protected function yieldIfExpiringResource(DrydockResource $resource) {
if (!$resource->canReceiveCommands()) {
return;
}
$this->yieldIfExpiring($resource->getUntil());
}
private function yieldIfExpiring($expires) {
if (!$expires) {
return;
}
if (!$this->getTaskDataValue('isExpireTask')) {
return;
}
$now = PhabricatorTime::getNow();
throw new PhabricatorWorkerYieldException($expires - $now);
}
protected function isTemporaryException(Exception $ex) {
if ($ex instanceof PhabricatorWorkerYieldException) {
return true;
}
if ($ex instanceof DrydockSlotLockException) {
return true;
}
if ($ex instanceof PhutilAggregateException) {
$any_temporary = false;
foreach ($ex->getExceptions() as $sub) {
if ($this->isTemporaryException($sub)) {
$any_temporary = true;
break;
}
}
if ($any_temporary) {
return true;
}
}
if ($ex instanceof PhutilProxyException) {
return $this->isTemporaryException($ex->getPreviousException());
}
return false;
}
protected function getYieldDurationFromException(Exception $ex) {
if ($ex instanceof PhabricatorWorkerYieldException) {
return $ex->getDuration();
}
if ($ex instanceof DrydockSlotLockException) {
return 5;
}
return 15;
}
protected function flushDrydockTaskQueue() {
// NOTE: By default, queued tasks are not scheduled if the current task
// fails. This is a good, safe default behavior. For example, it can
// protect us from executing side effect tasks too many times, like
// sending extra email.
// However, it is not the behavior we want in Drydock, because we queue
// followup tasks after lease and resource failures and want them to
// execute in order to clean things up.
// At least for now, we just explicitly flush the queue before exiting
// with a failure to make sure tasks get queued up properly.
try {
$this->flushTaskQueue();
} catch (Exception $ex) {
// If this fails, we want to swallow the exception so the caller throws
// the original error, since we're more likely to be able to understand
// and fix the problem if we have the original error than if we replace
// it with this one.
phlog($ex);
}
return $this;
}
protected function canReclaimResource(DrydockResource $resource) {
$viewer = $this->getViewer();
// Don't reclaim a resource if it has been updated recently. If two
// leases are fighting, we don't want them to keep reclaiming resources
// from one another forever without making progress, so make resources
// immune to reclamation for a little while after they activate or update.
// TODO: It would be nice to use a more narrow time here, like "last
// activation or lease release", but we don't currently store that
// anywhere.
$updated = $resource->getDateModified();
$now = PhabricatorTime::getNow();
$ago = ($now - $updated);
if ($ago < phutil_units('3 minutes in seconds')) {
return false;
}
$statuses = array(
DrydockLeaseStatus::STATUS_PENDING,
DrydockLeaseStatus::STATUS_ACQUIRED,
DrydockLeaseStatus::STATUS_ACTIVE,
DrydockLeaseStatus::STATUS_RELEASED,
DrydockLeaseStatus::STATUS_BROKEN,
);
// Don't reclaim resources that have any active leases.
$leases = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withResourcePHIDs(array($resource->getPHID()))
->withStatuses($statuses)
->setLimit(1)
->execute();
if ($leases) {
return false;
}
return true;
}
protected function reclaimResource(
DrydockResource $resource,
DrydockLease $lease) {
$viewer = $this->getViewer();
+ // When the resource releases, we we want to reawaken this task since it
+ // should be able to start building a new resource right away.
+ $worker_task_id = $this->getCurrentWorkerTaskID();
+
$command = DrydockCommand::initializeNewCommand($viewer)
->setTargetPHID($resource->getPHID())
->setAuthorPHID($lease->getPHID())
->setCommand(DrydockCommand::COMMAND_RECLAIM)
+ ->setProperty('awakenTaskIDs', array($worker_task_id))
->save();
$resource->scheduleUpdate();
return $this;
}
}

File Metadata

Mime Type
text/x-diff
Expires
Mon, Apr 28, 2:40 AM (16 h, 35 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
107674
Default Alt Text
(19 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)