/** * @return Scalr_Net_Ssh2_Client * Enter description here ... */ public function GetSsh2Client() { $ssh2Client = new Scalr_Net_Ssh2_Client(); switch ($this->platform) { case SERVER_PLATFORMS::RACKSPACE: $ssh2Client->addPassword('root', $this->GetProperty(RACKSPACE_SERVER_PROPERTIES::ADMIN_PASS)); break; case SERVER_PLATFORMS::EC2: $userName = '******'; // Temporary server for role builder if ($this->status == SERVER_STATUS::TEMPORARY) { $keyName = 'SCALR-ROLESBUILDER'; } else { $keyName = "FARM-{$this->farmId}"; } try { $key = Scalr_Model::init(Scalr_Model::SSH_KEY)->loadGlobalByName($keyName, $this->GetProperty(EC2_SERVER_PROPERTIES::REGION), $this->envId); if (!$key) { throw new Exception(_("There is no SSH key for server: {$this->serverId}")); } } catch (Exception $e) { throw new Exception("Cannot init SshKey object: {$e->getMessage()}"); } $priv_key_file = tempnam("/tmp", "AWSK"); @file_put_contents($priv_key_file, $key->getPrivate()); $pub_key_file = tempnam("/tmp", "AWSK"); @file_put_contents($pub_key_file, $key->getPublic()); $ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file); break; } return $ssh2Client; }
public function rndcStatus() { $retval = $this->ssh2Client->exec("{$this->rndcPath} status"); $this->logger->info("Execute rndc status: {$retval}"); preg_match_all("/number of zones:[^0-9]([0-9]+)/", $retval, $matches); if ($matches[1][0] > 0) { return $matches[1][0]; } else { return false; } }
/** * Upload S3cmd config file, AWS private key and certificate to instance aftre instance boot. * Also execute hostInit hooks from hooks/hostInit folder * * @param array $instanceinfo * @param string $local_ip * @param string $remote_ip * @param string $public_key */ public function OnHostInit(HostInitEvent $event) { if ($event->DBServer->IsSupported("0.5")) { $this->Logger->info("Scalarizr instance. Skipping SSH observer..."); return true; } if ($event->DBServer->platform != SERVER_PLATFORMS::EC2) { return true; } // Get farm info and client info from database; $DBFarm = DBFarm::LoadByID($this->FarmID); $DBRole = DBRole::loadById($event->DBServer->roleId); // Get Role info $ssh_port = $DBRole->getProperty(DBRole::PROPERTY_SSH_PORT) ? $DBRole->getProperty(DBRole::PROPERTY_SSH_PORT) : 22; // Generate s3cmd config file $s3cfg = CONFIG::$S3CFG_TEMPLATE; $s3cfg = str_replace("[access_key]", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::ACCESS_KEY), $s3cfg); $s3cfg = str_replace("[secret_key]", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::SECRET_KEY), $s3cfg); $s3cfg = str_replace("\r\n", "\n", $s3cfg); // Prepare public key for SSH connection $pub_key_file = tempnam("/tmp", "AWSK"); $res = file_put_contents($pub_key_file, $event->PublicKey); $this->Logger->debug("Creating temporary file for public key: {$res}"); try { $key = Scalr_Model::init(Scalr_Model::SSH_KEY)->loadGlobalByFarmId($event->DBServer->farmId, $event->DBServer->GetFarmRoleObject()->CloudLocation); if (!$key) { throw new Exception(_("There is no SSH key for server: {$event->DBServer->serverId}")); } } catch (Exception $e) { throw new Exception("Cannot init SshKey object: {$e->getMessage()}"); } // Prepare private key for SSH connection $priv_key_file = tempnam("/tmp", "AWSK"); $res = file_put_contents($priv_key_file, $key->getPrivate()); $this->Logger->debug("Creating temporary file for private key: {$res}"); // Connect to SSH $SSH2 = new Scalr_Net_Ssh2_Client(); $SSH2->addPubkey("root", $pub_key_file, $priv_key_file); if ($SSH2->connect($event->ExternalIP, $ssh_port)) { // Upload keys and s3 config to instance $res = $SSH2->sendFile("/etc/aws/keys/pk.pem", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), "w+", false); $res2 = $SSH2->sendFile("/etc/aws/keys/cert.pem", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE), "w+", false); $res3 = $SSH2->sendFile("/etc/aws/keys/s3cmd.cfg", $s3cfg, "w+", false); // remove temporary files @unlink($pub_key_file); @unlink($priv_key_file); } else { // remove temporary files @unlink($pub_key_file); @unlink($priv_key_file); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($this->FarmID, "Cannot upload ec2 keys to '{$event->DBServer->serverId}' instance. Failed to connect to SSH '{$event->ExternalIP}:{$ssh_port}'")); throw new Exception("Cannot upload keys on '{$event->DBServer->serverId}'. Failed to connect to '{$event->ExternalIP}:{$ssh_port}'."); } }
/** * @return Scalr_Net_Ssh2_Client * Enter description here ... */ public function GetSsh2Client() { $ssh2Client = new Scalr_Net_Ssh2_Client(); switch ($this->platform) { case SERVER_PLATFORMS::RACKSPACENG_UK: case SERVER_PLATFORMS::RACKSPACENG_US: $ssh2Client->addPassword('root', $this->GetProperty(OPENSTACK_SERVER_PROPERTIES::ADMIN_PASS)); break; case SERVER_PLATFORMS::RACKSPACE: $ssh2Client->addPassword('root', $this->GetProperty(RACKSPACE_SERVER_PROPERTIES::ADMIN_PASS)); break; case SERVER_PLATFORMS::GCE: $userName = '******'; if ($this->status == SERVER_STATUS::TEMPORARY) { $keyName = 'SCALR-ROLESBUILDER-' . SCALR_ID; } else { $keyName = "FARM-{$this->farmId}-" . SCALR_ID; } try { $key = (new SshKey())->loadGlobalByName($this->envId, SERVER_PLATFORMS::GCE, "", $keyName); if (!$key) { throw new Exception(_("There is no SSH key for server: {$this->serverId}")); } } catch (Exception $e) { throw new Exception("Cannot init SshKey object: {$e->getMessage()}"); } $priv_key_file = tempnam("/tmp", "GCEPK"); @file_put_contents($priv_key_file, $key->privateKey); $this->tmpFiles[] = $priv_key_file; $pub_key_file = tempnam("/tmp", "GCEK"); @file_put_contents($pub_key_file, $key->publicKey); $this->tmpFiles[] = $pub_key_file; $ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file); break; case SERVER_PLATFORMS::IDCF: case SERVER_PLATFORMS::EC2: $userName = '******'; $skipKeyValidation = false; // Temporary server for role builder $sshKey = new SshKey(); if ($this->status == SERVER_STATUS::TEMPORARY) { $keyName = "SCALR-ROLESBUILDER-" . SCALR_ID . "-{$this->envId}"; if (!$sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $keyName)) { $keyName = "SCALR-ROLESBUILDER-" . SCALR_ID; } try { $bundleTaskId = $this->GetProperty(\SERVER_PROPERTIES::SZR_IMPORTING_BUNDLE_TASK_ID); $bundleTask = BundleTask::LoadById($bundleTaskId); if ($bundleTask->osFamily == 'amazon') { $userName = '******'; } } catch (Exception $e) { } } else { $keyName = "FARM-{$this->farmId}-" . SCALR_ID; $oldKeyName = "FARM-{$this->farmId}"; $key = $sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $oldKeyName); if ($key) { $keyName = $oldKeyName; $skipKeyValidation = true; } } if (!$skipKeyValidation) { try { $key = $sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $keyName); if (!$key) { throw new Exception(sprintf('Could not find SSH Key for server "%s" with name:"%s", cloud-location:"%s", platform:"%s", environment:"%d".', $this->serverId, $keyName, $this->GetCloudLocation(), $this->platform, $this->envId)); } } catch (Exception $e) { throw new Exception("Cannot init SshKey object: {$e->getMessage()}"); } } $priv_key_file = tempnam("/tmp", "AWSK"); @file_put_contents($priv_key_file, $key->privateKey); $this->tmpFiles[] = $priv_key_file; $pub_key_file = tempnam("/tmp", "AWSK"); $this->tmpFiles[] = $pub_key_file; $pubKey = $key->publicKey; if (!stristr($pubKey, $keyName)) { $pubKey .= " {$keyName}"; } @file_put_contents($pub_key_file, $pubKey); $ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file); break; } return $ssh2Client; }
function handleWork($farmId) { $this->cleanup(); $DBFarm = DBFarm::LoadByID($farmId); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; $this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})"); // // Collect information from database // $servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED)); $this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database"); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { return; } /* O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}} O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}} */ foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { try { if ($DBServer->status != SERVER_STATUS::PENDING) { $p = PlatformFactory::NewPlatform($DBServer->platform); if (!$p->IsServerExists($DBServer)) { if ($DBServer->platform == SERVER_PLATFORMS::RACKSPACE) { if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) { //TODO: Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed. RACKSPACE, doing nothing.", $DBServer->serverId))); mail("*****@*****.**", "RACKSPACE - {$DBServer->serverId} ({$DBServer->remoteIp}) ({$DBServer->GetCloudServerID()}) ({$DBServer->GetProperty(RACKSPACE_SERVER_PROPERTIES::HOST_ID)})", serialize($p->_tmpVar)); $DBServer->SetProperty("rackspace.crashed", "1"); continue; } } else { if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); // Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId))); Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); continue; } } } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials")) { continue; } if (stristr($e->getMessage(), "Could not connect to host")) { continue; } print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; continue; } try { if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->GetRealStatus()->isTerminated()) { if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName()))); } $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->GetRealStatus()->IsRunning() && $DBServer->status != SERVER_STATUS::RUNNING) { if ($DBServer->status != SERVER_STATUS::TERMINATED) { if ($DBServer->platform == SERVER_PLATFORMS::RDS) { //TODO: timeouts if ($DBServer->status == SERVER_STATUS::PENDING) { $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $event = new HostInitEvent($DBServer, $info['localIp'], $info['remoteIp'], ''); } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = new HostUpEvent($DBServer, ""); // TODO: add mysql replication password } if ($event) { Scalr::FireEvent($DBServer->farmId, $event); } else { //TODO: Log } } else { if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA) { if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED)) { $dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject(); $ssh2Client = new Scalr_Net_Ssh2_Client(); $ssh2Client->addPassword($dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER), $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS)); $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT); if (!$port) { $port = 22; } try { $ssh2Client->connect($info['remoteIp'], $port); foreach ($DBServer->GetCloudUserData() as $k => $v) { $u_data .= "{$k}={$v};"; } $u_data = trim($u_data, ";"); $ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false); $DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage())); } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 300; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $DBServer->status = SERVER_STATUS::TERMINATED; $DBServer->Save(); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event) { $scripting_timeout = (int) $this->db->GetOne("SELECT sum(timeout) FROM farm_role_scripts \n\t\t\t\t\t\t\t\t\t\tWHERE event_name=? AND \n\t\t\t\t\t\t\t\t\t\tfarm_roleid=? AND issync='1'", array($scripting_event, $DBServer->farmRoleId)); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time()) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, "Server '{$DBServer->serverId}' did not send '{$event}' event in {$launch_timeout} seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.")); try { Scalr::FireEvent($DBFarm->ID, new BeforeHostTerminateEvent($DBServer, false)); } catch (Exception $err) { $this->logger->fatal($err->getMessage()); } } } // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'])); } //TODO: Check health: } } } } elseif ($DBServer->GetRealStatus()->isRunning() && $DBServer->status == SERVER_STATUS::RUNNING) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'])); } //TODO: Check health: } else { //TODO: Check reboot timeout } } if ($DBServer->status == SERVER_STATUS::PENDING_TERMINATE || $DBServer->status == SERVER_STATUS::TERMINATED) { if ($DBServer->status == SERVER_STATUS::TERMINATED || !$DBServer->dateShutdownScheduled || $DBServer->dateShutdownScheduled && strtotime($DBServer->dateShutdownScheduled) + 60 * 3 < time()) { try { if (!$DBServer->GetRealStatus()->isTerminated()) { try { if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED))); if ($DBServer->index == 1 && $serversCount > 1) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform))); continue; } } } catch (Exception $e) { } Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Terminating server '%s' (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform))); PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer); $this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Server is running on Amazon but has terminated status on Scalr"), $DBServer->serverId)); } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Role was removed from farm"), $DBServer->serverId)); $DBServer->Remove(); } elseif (stristr($e->getMessage(), "disableApiTermination")) { continue; } else { throw $e; } } } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { var_dump($e); } else { print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } } } }