There are 3 queues in redis.
If I insert a data at a same time with different browsers, both requests are accepted.
I get 2 duplicated records. Any helps?
For example, the first request is on the 1st queue.
At the second request, 2nd queue is searched for finding the data.
But the data doesn't exist in the queue.
The second request is inserted on the queue.
Is it understandable?
public function save($evt_no, $type = "redis") {
$name = '';
$res = true;
try{
$headers = getallheaders();
$bodys = $_REQUEST;
$session = $_SESSION;
foreach ($_FILES as $fileKey=>$fileVal) {
if ($fileVal['error'] == 0) {
try {
$uploaded_row = $this->fileL->upload('queueEventParticipant', $fileKey, array() ,'queue');
} catch (\Exception $ex) {
throw $ex;
}
$bodys['file'][_return_numeric($fileKey)] = $uploaded_row;
}
}
$data = array(
'header' => $headers,
'body' => $bodys,
'session' => $session
);
$data['body']['evt_no'] = $evt_no;
$data = json_encode($data);
if($type == "redis"){
$name = $this->attendQueueRedisService->setNewRsvp($data);
} else {
throw new \Exception("No exist");
}
} catch (\Exception $ex){
log_message("error", $ex->getMessage());
throw $ex;
}
if($res){
return $name;
} else {
return "Error";
}
}
class AttendQueueRedisService extends CI_Model {
private $redisClient;
private $redisConfig;
const PREFIX_DONE = 'done_';
const PREFIX_ERROR = 'error_';
public function __construct() {
parent::__construct();
if ($this->load->config('redis', true, true)) {
$this->redisConfig = $this->config->config['redis'];
}
$this->redisClient = new Redis();
$this->redisClient->connect($this->redisConfig['hostname'], $this->redisConfig['port']);
$this->redisClient->auth($this->redisConfig['auth']);
$this->redisClient->select($this->redisConfig['queue']);
}
public function setNewRsvp($data) {
$key = uniqid('rsvp_'.gethostname().'_',true).':redis';
$this->redisClient->set($key, $data, 60 * 30);
return $key;
}
}
heres my api.php file code in try body else case is taking too much time
<?php
require_once(dirname(__FILE__) . "/constants.php");
includeMonstaConfig();
session_start();
require_once(dirname(__FILE__) . '/lib/helpers.php');
require_once(dirname(__FILE__) . '/lib/response_helpers.php');
require_once(dirname(__FILE__) . '/request_processor/RequestMarshaller.php');
if (file_exists(dirname(__FILE__) . '/../../mftp_extensions.php')) {
include_once(dirname(__FILE__) . '/../../mftp_extensions.php');
}
dieIfNotPOST();
require_once(dirname(__FILE__) . '/lib/access_check.php');
$marshaller = new RequestMarshaller();
try {
$request = json_decode($_POST['request'], true);
if ($request['actionName'] == 'fetchFile' || $request['actionName'] == 'downloadMultipleFiles') {
switch ($request['actionName']) {
case 'fetchFile':
$outputPath = $marshaller->prepareFileForFetch($request);
$outputFileName = monstaBasename($request['context']['remotePath']);
break;
case 'downloadMultipleFiles':
$outputResponse = $marshaller->marshallRequest($request, false, true);
$outputPath = $outputResponse["data"];
$outputFileName = "mftp_zip_" . date("Y_m_d_H_i_s") . ".zip";
}
$fileKey = generateRandomString(16);
$_SESSION[MFTP_SESSION_KEY_PREFIX . $fileKey] = array(
"path" => $outputPath,
"fileName" => $outputFileName
);
$response = array(
"success" => true,
"fileKey" => $fileKey
);
print json_encode($response);
} else {
$skipConfigurationActions = array('checkSavedAuthExists', 'writeSavedAuth', 'readSavedAuth',
'readLicense', 'getSystemVars', 'resetPassword', 'forgotPassword', 'validateSavedAuthPassword',
'downloadLatestVersionArchive', 'installLatestVersion');
$skipConfiguration = in_array($request['actionName'], $skipConfigurationActions);
$serializedResponse = $marshaller->marshallRequest($request, $skipConfiguration);
print $serializedResponse;
}
} catch (Exception $e) {
$marshaller->disconnect();
handleExceptionInRequest($e);
}
$marshaller->disconnect();
here is the RequestMarshaller.php code showing only the request code
<?php
require_once(dirname(__FILE__) . '/RequestDispatcher.php');
require_once(dirname(__FILE__) . "/../lib/helpers.php");
require_once(dirname(__FILE__) . "/../system/ApplicationSettings.php");
class RequestMarshaller {
/**
* #var RequestDispatcher
*/
private $requestDispatcher;
private function initRequestDispatcher($request, $skipConfiguration = false) {
if(!$skipConfiguration)
$request['configuration'] = $this->applyConnectionRestrictions($request['connectionType'],
$request['configuration']);
if (is_null($this->requestDispatcher))
$this->requestDispatcher = new RequestDispatcher($request['connectionType'], $request['configuration'],
null, null, $skipConfiguration);
}
public function marshallRequest($request, $skipConfiguration = false, $skipEncode = false) {
$this->initRequestDispatcher($request, $skipConfiguration);
$response = array();
if ($request['actionName'] == 'putFileContents')
$response = $this->putFileContents($request);
else if ($request['actionName'] == 'getFileContents')
$response = $this->getFileContents($request);
else {
$context = array_key_exists('context', $request) ? $request['context'] : null;
$responseData = $this->requestDispatcher->dispatchRequest($request['actionName'], $context);
$response['success'] = true;
if(is_object($responseData)) {
$response['data'] = method_exists($responseData, 'legacyJsonSerialize') ?
$responseData->legacyJsonSerialize() : $responseData;
} else
$response['data'] = $responseData;
}
if ($skipEncode)
return $response;
return json_encode($response);
}
}
}
and here is my RequestDispatcher.php code
<?php
class RequestDispatcher {
/**
* #var ConnectionBase
*/
private $connection;
/**
* #var string
*/
private $connectionType;
/**
* #var array
*/
private $rawConfiguration;
public $username;
public function getusername(){
$configuration = $this->connection->getConfiguration();
$this->username=$configuration->getRemoteUsername();
return $this->username;
}
public function __construct($connectionType, $rawConfiguration, $configurationFactory = null,
$connectionFactory = null, $skipConfiguration = false) {
$this->connectionType = $connectionType;
/* allow factory objects to be passed in for testing with mocks */
if ($skipConfiguration) {
$this->connection = null;
} else {
$this->rawConfiguration = $rawConfiguration;
$configurationFactory = is_null($configurationFactory) ? new ConfigurationFactory() : $configurationFactory;
$connectionFactory = is_null($connectionFactory) ? new ConnectionFactory() : $connectionFactory;
$configuration = $configurationFactory->getConfiguration($connectionType, $rawConfiguration);
$this->connection = $connectionFactory->getConnection($connectionType, $configuration);
}
}
public function dispatchRequest($actionName, $context = null) {
if (in_array($actionName, array(
'listDirectory',
'downloadFile',
'uploadFile',
'deleteFile',
'makeDirectory',
'deleteDirectory',
'rename',
'changePermissions',
'copy',
'testConnectAndAuthenticate',
'checkSavedAuthExists',
'writeSavedAuth',
'readSavedAuth',
'readLicense',
'getSystemVars',
'fetchRemoteFile',
'uploadFileToNewDirectory',
'downloadMultipleFiles',
'createZip',
'setApplicationSettings',
'deleteMultiple',
'extractArchive',
'updateLicense',
'reserveUploadContext',
'transferUploadToRemote',
'getRemoteFileSize',
'getDefaultPath',
'downloadForExtract',
'cleanUpExtract',
'resetPassword',
'forgotPassword',
'validateSavedAuthPassword',
'downloadLatestVersionArchive',
'installLatestVersion'
))) {
if (!is_null($context))
return $this->$actionName($context);
else
return $this->$actionName();
}
throw new InvalidArgumentException("Unknown action $actionName");
}
public function getConnection() {
return $this->connection;
}
private function connectAndAuthenticate($isTest = false) {
$sessionNeedsStarting = false;
if (function_exists("session_status")) {
if (session_status() == PHP_SESSION_NONE) {
$sessionNeedsStarting = true;
}
} else {
$sessionNeedsStarting = session_id() == "";
}
$configuration = $this->connection->getConfiguration();
$this->username=$configuration->getRemoteUsername();
if ($sessionNeedsStarting && !defined("MONSTA_UNIT_TEST_MODE")) { // TODO: pass in this as parameter to avoid global state
session_start();
$_SESSION["RemoteUsername"]=$configuration->getRemoteUsername();
}
$maxFailures = defined("MFTP_MAX_LOGIN_FAILURES") ? MFTP_MAX_LOGIN_FAILURES : 0;
$loginFailureResetTimeSeconds = defined("MFTP_LOGIN_FAILURES_RESET_TIME_MINUTES")
? MFTP_LOGIN_FAILURES_RESET_TIME_MINUTES * 60 : 0;
if (!isset($_SESSION["MFTP_LOGIN_FAILURES"]))
$_SESSION["MFTP_LOGIN_FAILURES"] = array();
$banManager = new UserBanManager($maxFailures, $loginFailureResetTimeSeconds,
$_SESSION["MFTP_LOGIN_FAILURES"]);
if ($banManager->hostAndUserBanned($configuration->getHost(), $configuration->getRemoteUsername())) {
mftpActionLog("Log in", $this->connection, "", "", "Login and user has exceed maximum failures.");
throw new FileSourceAuthenticationException("Login and user has exceed maximum failures.",
LocalizableExceptionDefinition::$LOGIN_FAILURE_EXCEEDED_ERROR, array(
"banTimeMinutes" => MFTP_LOGIN_FAILURES_RESET_TIME_MINUTES
));
}
try {
$this->connection->connect();
} catch (Exception $e) {
mftpActionLog("Log in", $this->connection, "", "", $e->getMessage());
throw $e;
}
try {
$this->connection->authenticate();
} catch (Exception $e) {
mftpActionLog("Log in", $this->connection, "", "", $e->getMessage());
$banManager->recordHostAndUserLoginFailure($configuration->getHost(),
$configuration->getRemoteUsername());
$_SESSION["MFTP_LOGIN_FAILURES"] = $banManager->getStore();
throw $e;
}
$banManager->resetHostUserLoginFailure($configuration->getHost(), $configuration->getRemoteUsername());
$_SESSION["MFTP_LOGIN_FAILURES"] = $banManager->getStore();
if ($isTest) {
// only log success if it is the first connect from the user
mftpActionLog("Log in", $this->connection, "", "", "");
}
if ($configuration->getInitialDirectory() === "" || is_null($configuration->getInitialDirectory())) {
return $this->connection->getCurrentDirectory();
}
return null;
}
public function disconnect() {
if ($this->connection != null && $this->connection->isConnected())
$this->connection->disconnect();
}
public function listDirectory($context) {
$this->connectAndAuthenticate();
$directoryList = $this->connection->listDirectory($context['path'], $context['showHidden']);
$this->disconnect();
return $directoryList;
}
public function downloadFile($context, $skipLog = false) {
$this->connectAndAuthenticate();
$transferOp = TransferOperationFactory::getTransferOperation($this->connectionType, $context);
$this->connection->downloadFile($transferOp);
if (!$skipLog) {
// e.g. if editing a file don't log that it was also downloaded
mftpActionLog("Download file", $this->connection, dirname($transferOp->getRemotePath()), monstaBasename($transferOp->getRemotePath()), "");
}
$this->disconnect();
}
public function downloadMultipleFiles($context) {
$this->connectAndAuthenticate();
$fileFinder = new RecursiveFileFinder($this->connection, $context['baseDirectory']);
$foundFiles = $fileFinder->findFilesInPaths($context['items']);
foreach ($foundFiles as $foundFile) {
$fullPath = PathOperations::join($context['baseDirectory'], $foundFile);
mftpActionLog("Download file", $this->connection, dirname($fullPath), monstaBasename($fullPath), "");
}
$zipBuilder = new ZipBuilder($this->connection, $context['baseDirectory']);
$zipPath = $zipBuilder->buildZip($foundFiles);
$this->disconnect();
return $zipPath;
}
public function createZip($context) {
$this->connectAndAuthenticate();
$fileFinder = new RecursiveFileFinder($this->connection, $context['baseDirectory']);
$foundFiles = $fileFinder->findFilesInPaths($context['items']);
foreach ($foundFiles as $foundFile) {
$fullPath = PathOperations::join($context['baseDirectory'], $foundFile);
mftpActionLog("Download file", $this->connection, dirname($fullPath), monstaBasename($fullPath), "");
}
$zipBuilder = new ZipBuilder($this->connection, $context['baseDirectory']);
$destPath = PathOperations::join($context['baseDirectory'], $context['dest']);
$zipPath = $zipBuilder->buildLocalZip($foundFiles, $destPath);
$this->connection->uploadFile(new FTPTransferOperation($zipPath, $destPath, FTP_BINARY));
$this->disconnect();
return $zipPath;
}
public function uploadFile($context, $preserveRemotePermissions = false, $skipLog = false) {
$this->connectAndAuthenticate();
$transferOp = TransferOperationFactory::getTransferOperation($this->connectionType, $context);
$this->connection->uploadFile($transferOp, $preserveRemotePermissions);
if (!$skipLog) {
// e.g. if editing a file don't log that it was also uploaded
mftpActionLog("Upload file", $this->connection, dirname($transferOp->getRemotePath()), monstaBasename($transferOp->getRemotePath()), "");
}
$this->disconnect();
}
public function uploadFileToNewDirectory($context) {
// This will first create the target directory if it doesn't exist and then upload to that directory
$this->connectAndAuthenticate();
$transferOp = TransferOperationFactory::getTransferOperation($this->connectionType, $context);
$this->connection->uploadFileToNewDirectory($transferOp);
mftpActionLog("Upload file", $this->connection, dirname($transferOp->getRemotePath()), monstaBasename($transferOp->getRemotePath()), "");
$this->disconnect();
}
public function deleteFile($context) {
$this->connectAndAuthenticate();
$this->connection->deleteFile($context['remotePath']);
mftpActionLog("Delete file", $this->connection, dirname($context['remotePath']), monstaBasename($context['remotePath']), "");
$this->disconnect();
}
public function makeDirectory($context) {
$this->connectAndAuthenticate();
$this->connection->makeDirectory($context['remotePath']);
$this->disconnect();
}
public function deleteDirectory($context) {
$this->connectAndAuthenticate();
$this->connection->deleteDirectory($context['remotePath']);
$this->disconnect();
}
public function rename($context) {
$this->connectAndAuthenticate();
if(array_key_exists('action', $context) && $context['action'] == 'move') {
$action = 'Move';
} else {
$action = 'Rename';
}
$itemType = $this->connection->isDirectory($context['source']) ? 'folder' : 'file';
$this->connection->rename($context['source'], $context['destination']);
if ($action == 'Move') {
mftpActionLog($action . " " . $itemType, $this->connection, dirname($context['source']),
monstaBasename($context['source']) . " to " . $context['destination'],
"");
}
if ($action == 'Rename') {
mftpActionLog($action . " " . $itemType, $this->connection, dirname($context['source']),
monstaBasename($context['source']) . " to " . monstaBasename($context['destination']),
"");
}
$this->disconnect();
}
public function changePermissions($context) {
$this->connectAndAuthenticate();
$itemType = $this->connection->isDirectory($context['remotePath']) ? 'folder' : 'file';
$this->connection->changePermissions($context['mode'], $context['remotePath']);
mftpActionLog("CHMOD " . $itemType, $this->connection, dirname($context['remotePath']),
monstaBasename($context['remotePath']) . " to " . decoct($context['mode']), "");
$this->disconnect();
}
public function copy($context) {
$this->connectAndAuthenticate();
$this->connection->copy($context['source'], $context['destination']);
$this->disconnect();
}
public function testConnectAndAuthenticate($context, $isInitalLogin = true) {
$initialDirectory = $this->connectAndAuthenticate($isInitalLogin);
$serverCapabilities = array("initialDirectory" => $initialDirectory);
if (isset($context['getServerCapabilities']) && $context['getServerCapabilities']) {
$serverCapabilities["changePermissions"] = $this->connection->supportsPermissionChange();
}
clearOldTransfers();
return array("serverCapabilities" => $serverCapabilities);
}
public function checkSavedAuthExists() {
if ($this->readLicense() == null)
return false;
return AuthenticationStorage::configurationExists(AUTHENTICATION_FILE_PATH);
}
public function writeSavedAuth($context) {
if ($this->readLicense() == null)
return;
AuthenticationStorage::saveConfiguration(AUTHENTICATION_FILE_PATH, $context['password'],
$context['authData']);
}
public function readSavedAuth($context) {
if ($this->readLicense() == null)
return array();
return AuthenticationStorage::loadConfiguration(AUTHENTICATION_FILE_PATH, $context['password']);
}
public function readLicense() {
$keyPairSuite = new KeyPairSuite(PUBKEY_PATH);
$licenseReader = new LicenseReader($keyPairSuite);
$license = $licenseReader->readLicense(MONSTA_LICENSE_PATH);
if (is_null($license))
return $license;
$publicLicenseKeys = array("expiryDate", "version", "isTrial", "licenseVersion", "productEdition");
$publicLicense = array();
foreach ($publicLicenseKeys as $publicLicenseKey) {
if (isset($license[$publicLicenseKey]))
$publicLicense[$publicLicenseKey] = $license[$publicLicenseKey];
}
return $publicLicense;
}
private function recordAffiliateSource($licenseEmail) {
$affiliateChecker = new AffiliateChecker();
$installUrl = getMonstaInstallUrl();
$affiliateId = defined("MFTP_AFFILIATE_ID") ? MFTP_AFFILIATE_ID : "";
return $affiliateChecker->recordAffiliateSource($affiliateId, $licenseEmail, $installUrl);
}
public function getSystemVars() {
$systemVars = SystemVars::getSystemVarsArray();
$applicationSettings = new ApplicationSettings(APPLICATION_SETTINGS_PATH);
$systemVars['applicationSettings'] = $applicationSettings->getSettingsArray();
return $systemVars;
}
public function setApplicationSettings($context) {
$applicationSettings = new ApplicationSettings(APPLICATION_SETTINGS_PATH);
$applicationSettings->setFromArray($context['applicationSettings']);
$applicationSettings->save();
}
public function validateSavedAuthPassword($context) {
return AuthenticationStorage::validateAuthenticationPassword(AUTHENTICATION_FILE_PATH, $context["password"]);
}
}
i am not that much experienced in this field i am a beginner and got this task to find out the cause of the delay and resolve it . kindly help me in this regard i have tried many things as i thought that json_encoding(response) might be the cause but just 10 items are returning in response
I'm setting up a rest-API on my server, and I want to update a table (i.e "comp_holding_stock"). but every time I test to post new data it returns "No item found"
Here is my controller
public function create_comp_holding_stock(){
$returnArr['status'] = '0';
$returnArr['response'] = '';
try {
if (!$this->input->post()) {
$returnArr['response'] = "Only POST method is allowed";
} else {
$holding_stock_data = array(
'comp_id' => $this->input->post('comp_id'),
'customer_id' => $this->input->post('customer_id'),
'quantity' => $this->input->post('quantity'),
'date' => date('Y-m-d H:i:s')
);
if (!isset($holding_stock_data)) {
$returnArr['response'] = "Some Parameters are missing";
} else {
$customer = $this->Customer->save_holding_stock($holding_stock_data);
if (!$customer) {
$returnArr['response'] = 'No items found';
} else {
$returnArr['status'] = '1';
$returnArr['response'] = $customer;
}
}
}
} catch (Exception $ex) {
$returnArr['response'] = "Error in connection";
$returnArr['error'] = $ex->getMessage();
}
$response = json_encode($returnArr, JSON_PRETTY_PRINT);
echo $response;
}
And here is my model below
public function save_holding_stock($holding_stock_data)
{
// $this->db->trans_start();
$success = $this->db->insert('comp_holding_stock', $holding_stock_data);
return $success;;
}
what am i doing wrong? what is the best approach to this scenarios
I would recommend try to check if you have load model in your controller.
And in your model try to do this.
public function save_holding_stock($holding_stock_data, $comp_id=FALSE)
{
if(!$comp_id == -1 || !$this->exists($comp_id))
{
if($this->db->insert('comp_holding_stock', $holding_stock_data))
{
$holding_stock_data['comp_id'] = $this->db->insert_id();
return TRUE;
}
return FALSE;
}
$this->db->where('comp_id', $comp_id);
return $this->db->update('comp_holding_stock', $holding_stock_data);
}
Try these changes in your code
In your controller,
$customer = $this->Customer->save_holding_stock($holding_stock_data);
$save_status = $this->db->affected_rows();
if ($save_status>0) {
$returnArr['status'] = '1';
$returnArr['response'] = $customer;
} else {
$returnArr['response'] = 'No items found';
}
In your model,
public function save_holding_stock($holding_stock_data)
{
// $this->db->trans_start();
$this->db->insert('comp_holding_stock', $holding_stock_data);
}
This is my code
class WcfClient {
public $wcfClient = null;
public $user = null;
public function __construct(){
if(isset($_SESSION['APIClient']) && $_SESSION['APIClient'] != null){
$this->wcfClient = $_SESSION['APIClient'];
}
}
public function __destruct(){
}
// Authanticate
private function Authenticate(){
global $_sogh_soapUrl, $_isDebug, $_sogh_header;
$wcargs = array();
$consumerAuthTicket = null;
if($this->wcfClient == null){
$args = array(
'clubname'=>'Wellness Institute at Seven Oaks',
'consumerName'=>'api',
'consumerPassword'=>'api'
);
try{
$wcargs = array(
'soap_version'=>SOAP_1_2
);
if($_isDebug){
$wcargs = array(
'soap_version'=>SOAP_1_2,
'proxy_host'=>"192.168.0.1",
'proxy_port'=>8080
);
}
// Connect to the API with soapclient
$soapAPIClient = new SoapClient($_sogh_soapUrl, $wcargs);
$response = $soapAPIClient->AuthenticateClubConsumer($args);
if(isset($response->AuthenticateClubConsumerResult)){
if(isset($response->AuthenticateClubConsumerResult->IsException) && $response->AuthenticateClubConsumerResult->IsException == true){
// some error occur
$this->wcfClient = null;
$_SESSION['APIClient'] = $this->wcfClient;
} else{
// set consumer ticket
$consumerAuthTicket = $response->AuthenticateClubConsumerResult->Value->AuthTicket;
// $loginData = $responseCode->ReturnValueOfConsumerLoginData;
$headers = array();
$headers[] = new SoapHeader($_sogh_header, "ConsumerAuthTicket", $consumerAuthTicket);
$soapAPIClient->__setSoapHeaders($headers);
// add to session
$this->wcfClient = $soapAPIClient;
$_SESSION['APIClient'] = $this->wcfClient;
}
}
} catch(SoapFault $fault){
$this->error('Fault: ' . $fault->faultcode . ' - ' . $fault->faultstring);
} catch(Exception $e){
$this->error('Error: ' . $e->getMessage());
}
}
return $this->wcfClient;
}
I store the soap client object in $_SESSION['APIClient'], but second times when run some data has been changed in session, I am use this class in drupal 7, I want to save the time using session, because authenticating takes long time.
Please help
Thank in advance
I have the simple app below. I'm turning off query logging in Laravel, I'm unsetting where possible, yet this function will only process about 800 records before I'm out of RAM on my 2GB Linode. I know I'm asking a lot of you guys but I can't seem to see where I'm leaking memory.
There are really only two major steps.
Step 1 - Move records from a temp table to production
class ListingMigrator
{
public function __construct($tempListing, $feed)
{
$this->tempListing = $tempListing;
$this->listing = $this->listingInstance();
$this->feed = $feed;
}
public static function migrateListing($listing, $feed)
{
$instance = new static($listing, $feed);
return $instance->migrate();
}
public function migrate()
{
$this->addExternalData();
$this->populateListing();
$this->processPhotos();
$this->deleteTempListing();
}
private function listingInstance()
{
DB::connection()->disableQueryLog();
$listing = Listing::findByMud($this->tempListing->matrix_unique_id);
return $listing ?: new Listing;
}
private function processPhotos()
{
$retsApi = new RetsFeedApi($this->feed);
/* Initialize Object */
$rets = $retsApi->findMostRecent();
$photos = $rets->getPhotosForListing($this->listing->matrix_unique_id);
foreach ($photos as $photo)
{
$uploader = new PhotoProcessor($this->listing, $photo);
$uploader->process();
}
}
private function populateListing()
{
DB::connection()->disableQueryLog();
$this->listing->fill($this->tempListing->toArray());
$this->listing->imported_at = $this->tempListing->created_at;
$this->listing->board = $this->tempListing->board;
return $this->listing->save();
}
private function addExternalData()
{
// Get Google lattitude and longitude
$googlecoords = getGoogleMapInfo($this->tempListing->FullAddress, $this->tempListing->City);
$this->listing->GoogleLat = $googlecoords['GoogleLat'];
$this->listing->GoogleLong = $googlecoords['GoogleLong'];
// Add or update the Subdivision Table (helper function)
$subdivisiondata = SubdivisionUpdate($this->tempListing->board, $this->tempListing->SubCondoName, $this->tempListing->Development);
$this->listing->SubdivisionID = $subdivisiondata['id'];
}
private function deleteTempListing()
{
return $this->tempListing->delete();
}
}
Step 2 - Download photos and reupload to Amazon S3
class PhotoProcessor
{
public function __construct(Listing $listing, $photoData)
{
$this->bucket = 'real-estate-listings';
$this->s3 = App::make('aws')->get('s3');
$this->tempFileName = 'app/storage/processing/images/retsphotoupload';
$this->photoData = $photoData;
$this->listing = $listing;
$this->photo = new RetsPhoto;
}
public function process()
{
$this->storeTempFile();
$this->storeFileInfo();
$this->buildPhoto();
$success = $this->pushToS3();
// if Result has the full URL or you want to build it, add it to $this->photo
DB::connection()->disableQueryLog();
$this->listing->photos()->save($this->photo);
$this->removeTempFile();
unset ($this->photoData);
return $success;
}
private function storeTempFile()
{
return File::put($this->tempFileName, $this->photoData['Data']) > 0;
}
private function storeFileInfo()
{
$fileInfo = getimagesize($this->tempFileName);
// Could even be its own object
$this->fileInfo = [
'width' => $fileInfo[0],
'height' => $fileInfo[1],
'mimetype' => $fileInfo['mime'],
'extension' => $this->getFileExtension($fileInfo['mime'])
];
}
private function buildPhoto()
{
$this->photo->number = $this->photoData['Object-ID']; // Storing this because it is relevant order wise
$this->photo->width = $this->fileInfo['width'];
$this->photo->height = $this->fileInfo['height'];
$this->photo->path = $this->getFilePath();
}
private function getFilePath()
{
$path = [];
if ($this->listing->City == NULL)
{
$path[] = Str::slug('No City');
}
else
{
$path[] = Str::slug($this->listing->City, $separator = '-');
}
if ($this->listing->Development == NULL)
{
$path[] = Str::slug('No Development');
}
else
{
$path[] = Str::slug($this->listing->Development, $separator = '-');
}
if ($this->listing->Subdivision == NULL)
{
$pathp[] = Str::slug('No Subdivision');
}
else
{
$path[] = Str::slug($this->listing->Subdivision, $separator = '-');
}
if ($this->listing->MLSNumber == NULL)
{
$pathp[] = Str::slug('No MLSNumber');
}
else
{
$path[] = Str::slug($this->listing->MLSNumber, $separator = '-');
}
$path[] = $this->photoData['Object-ID'].'.'.$this->fileInfo['extension'];
return strtolower(join('/', $path));
}
private function pushToS3()
{
return $this->s3->putObject([
'Bucket' => $this->bucket,
'Key' => $this->photo->path,
'ContentType'=> $this->fileInfo['mimetype'],
'SourceFile' => $this->tempFileName
]);
}
private function getFileExtension($mime)
{
// Use better algorithm than this
$ext = str_replace('image/', '', $mime);
return $ext == 'jpeg' ? 'jpg' : $ext;
}
private function removeTempFile()
{
return File::delete($this->tempFileName);
}
}
Edit to show RetsPhoto
class RetsPhoto extends Eloquent {
protected $table = 'rets_property_photos';
public function listing() {
return $this->belongsTo('Listing', 'matrix_unique_id', 'matrix_unique_id');
}
}
Edit #2: Chunk Call
This is in the app/command and the only thing in there is the fire() function below:
public function fire()
{
// Turn off query logging
DB::connection()->disableQueryLog();
$feeds = RetsFeed::where('active','=',1)->get();
foreach ($feeds as $feed)
{
$class = "TempListing{$feed->board}";
$listings = $class::orderBy('MatrixModifiedDT','desc');
$listings->chunk(50, function($listings) use($feed) {
$listings->each(function($listing) use ($feed) {
ListingMigrator::migrateListing($listing,$feed);
echo "Feed: $feed->board\r\n";
echo "SubcondoName: $listing->SubCondoName\r\n";
echo "Development: $listing->Development\r\n";
echo "\r\n";
});
});
}
}
I think I have figured it out.
Your system holds in memory all of the photo data. As witnessed by the unset ($this->photoData);
The problem is that you need to first complete the process function. Your application is not likely processing ANY photos so when you keep grabbing them from the file system you run out of memory BEFORE you even process a single one.
To Confirm this, simply grab 1 file not using the chunk method.
I am not very familar with Laravel, it could be grabbing all of the files all at once as well and eating the ram.
You can do some tracing with memory_get_usage(true) to find out exactly where the ram is getting eaten from. I would suggest analysing the fire method first.