(First problem is below)
Finally made it like i wanted ! What a headache ^^
Not using pool was the answer ;)
So, here is my code, output below !
As you can see on the output, i got my jobs distribute evenly between workers 1 & 2 while worker 0 is running a long job !
<?php
class ChildThread extends Threaded {
public $workerId;
protected $complete;
public function __construct($workerId){
// init some properties
$this->workerId = $workerId;
$this->complete = false;
}
public function run(){
// do some work
echo " STARTING " . $this->workerId . "\r\n";
if($this->workerId == 0){
sleep(10);
} else {
sleep(1);
}
echo " ENDING " . $this->workerId . "\r\n";
$this->complete = true;
}
public function isComplete() {
return $this->complete;
}
}
$WORKER_COUNT = 3;
$workers = array();
for ($i=0; $i<$WORKER_COUNT; $i++) {
$worker = new Worker();
$worker->start();
$workers[] = $worker;
}
$tasks = array();
//Create 9 jobs
for ($i=0; $i < 9; $i++) {
$tasks[] = new ChildThread($i);
}
$cptTasks = 0;
//References for running jobs
$taskss = array();
while(count($tasks) > 0){
foreach ($workers as $key => $worker) {
if($worker->isShutdown()){
echo "Shutdowned worker ".$key.", restarting it !"."\r\n";
$worker->start();
}
//If worker is free of charge
if($worker->isWorking() === false ){
//Check if task not null
if(isset($tasks[$cptTasks])){
echo ("Stacking job ".$cptTasks." on worker ".$key."\r\n");
$worker->stack($tasks[$cptTasks]);
$taskss[] = $tasks[$cptTasks];
//Clear job from jobs list
unset($tasks[$cptTasks]);
$cptTasks++;
}
}
}
usleep(2000);
}
//Clear workers
foreach ($workers as $key => $worker) {
echo "Shutdown worker : " .$key."\r\n";
$worker->shutdown();
}
?>
Output :
Stacking job 0 on worker 0
Stacking job 1 on worker 1
Stacking job 2 on worker 2
STARTING 1
STARTING 0
STARTING 2
ENDING 1
ENDING 2
Stacking job 3 on worker 1
Stacking job 4 on worker 2
STARTING 3
STARTING 4
ENDING 3
ENDING 4
Stacking job 5 on worker 1
Stacking job 6 on worker 2
STARTING 5
STARTING 6
ENDING 5
ENDING 6
Stacking job 7 on worker 1
Stacking job 8 on worker 2
STARTING 7
STARTING 8
Shutdown worker : 0
ENDING 8
ENDING 7
ENDING 0
Shutdown worker : 1
Shutdown worker : 2
I'm working on a pthread system, and i'm trying to implement kind of a pool system.
Currently, i'm creating a pool of 3 workers, then send my jobs with pool->submit().
Then, pool->collect()
And pool->shutdown()
But, i'm disapointed with the result.
Jobs seems to be distributed to all workers evenly
Worker 1 : jobs 1, 4, 7
Worker 2 : jobs 2, 5, 8
Worker 3 : jobs 3, 6, 9
Then, let's say my job 1 is pretty long, and all other are not dat long, my jobs will be finished like so :
2, 5, 3, 6, 8, 9, 1, 4, 7
But, what i'd like to achieve should be more like : 2, 3, 5, 4, 6, 8, 7, 9, 1
Like distributing jobs to the nearest not working worker, so my heavy job is running on worker 1, and all others jobs goes to workers 2 and 3
Is that's something doable ?
I'm i missing something here ?
Code exemple :
<?php
class ChildThread extends Threaded {
public $jobId;
protected $complete;
public function __construct($jobId){
// init some properties
$this->jobId = $jobId;
$this->complete = false;
}
public function run(){
echo "STARTING " . $this->jobId . "\r\n";
//Job ID 1 == heavy waiting
if($this->jobId == 1){
sleep(10);
} else {
sleep(1);
}
echo "ENDING " . $this->jobId . "\r\n";
$this->complete = true;
}
public function isComplete() {
return $this->complete;
}
}
$pool = new Pool(3);
$tasks = array();
// Add tasks to pool queue
for ($i=1; $i<=9; $i++) {
$task = new ChildThread($i);
$pool->submit($task);
$tasks[] = $task;
}
$pool->collect(function($work){
return $work->isComplete();
});
$pool->shutdown();
?>
Result :
STARTING 1
STARTING 2
STARTING 3
ENDING 2
STARTING 5
ENDING 3
STARTING 6
ENDING 5
STARTING 8
ENDING 6
STARTING 9
ENDING 8
ENDING 9
ENDING 1
STARTING 4
ENDING 4
STARTING 7
ENDING 7
Related
So I've been developing PHP for a bit now and have been attempting to troubleshoot why exec with a redirection of standard IO is still hanging the main thread.
exec(escapeshellcmd("php ".getcwd()."/orderProcessing.php" . " " . $Prepared . " " . escapeshellarg(35). "> /dev/null 2>&1 &"));
The code provided is what I've been trialing with, and I can't seem to get it to not hang until the other script completes. The reason I am doing this is in this paticular case, processing an order can take upto 30 seconds, and I don't want the user on the frontend to wait for that.
Can I only spawn child processes with php-fpm?
Is there something I have misconfigured?
Am I just misunderstanding how child processes work?
Setup is:
Centos 7 with Apache HTTPD and PHP 8.1.11
Any help appreciated!
Yes, it is possible in PHP. With proc_open() function, you can send command without wait another process. With handle opened stream, you can catch process status and check it consistently. For example:
$max = 5;
$streams = [];
$command = 'php ' . __DIR__ . '/../../process runSomeProcessCommand';
while (true) {
echo 'Working :' . count($streams) . "\n";
for ($i = 0; $i < $max; $i++) {
if (!isset($streams[$i])) {
$descriptor = [
0 => ['pipe', 'r'],
1 => ['pipe', 'w'],
2 => ['pipe', 'w']
];
echo "proc opened $i \n";
$streams[$i]['proc'] = proc_open($command . ":$i", $descriptor, $pipes);
$streams[$i]['pipes'] = $pipes;
$streams[$i]['ttl'] = time();
usleep(200000);
} else {
$info = proc_get_status($streams[$i]['proc']);
if ($info['running'] === false) {
echo "Finished $i . TTL: (" . (time() - $streams[$i]['ttl']) . " sec.).Response: " . stream_get_contents($streams[$i]['pipes'][1]) . " \n";
fclose($streams[$i]['pipes'][1]);
if ($error = stream_get_contents($streams[$i]['pipes'][2])) {
echo "Error for $i. Error: $error " . PHP_EOL;
fclose($streams[$i]['pipes'][2]);
}
proc_close($streams[$i]['proc']);
unset($streams[$i]);
} else {
echo "Running process (PID {$info['pid']}) - $i: \n";
}
# $return_value = proc_close($streams[$i]);
}
}
sleep(3);
}
There are 5 threads in same time which are doesn't wait one another.
Hi I'm having a problem trying to get threads to perform properly. The problem is that my php script will terminate unexpectedly at 1 of 2 specific points, during execution, and works the rest of the time.
I've created a test scenario that produces the same result:
This script creates 5 threads. Each thread adds up, each iteration, of a random number ranging from 10 to 20.
Secondly, Summary_Thread checks to see when all threads are completed before printing the summary... and there in lies the issue.
The script terminates during Summary_Thread::run and Stack_Item_Container_Stack::Compile_Summary, which is indirectly called at the end of Summary_Thread::run.
#!/usr/bin/php
<?php
ini_set('max_execution_time', 0);
ini_set('display_errors', 1);
error_reporting(E_ALL ^ E_NOTICE);
ignore_user_abort();
class Summary_Thread_Container
{
public $stack_item_container_stack;
private $summary_thread;
function __construct($thread_count, Stack_Item_Container_Stack $stack_item_container_stack)
{
$this->stack_item_container_stack = $stack_item_container_stack;
$this->summary_thread = new Summary_Thread($thread_count, $this);
$this->summary_thread->start();
}
public function Compile_Summary(){ $this->stack_item_container_stack->Compile_Summary(); }
}
class Summary_Thread extends Worker
{
private $summary_thread_container;
private $thread_count;
function __construct($thread_count, Summary_Thread_Container $summary_thread_container)
{
$this->summary_thread_container = $summary_thread_container;
$this->thread_count = $thread_count;
}
public function run()
{
$thread_count = 0;
echo "\n************************************** Stack Thread Count: {$this->thread_count} \n";
echo "*** START.\n";
if($this->thread_count == $thread_count)
echo "*** THREAD COUNTS MATCH.\n";
else
echo "*** THREAD COUNTS DO NOT MATCH.\n";
while($this->thread_count != $thread_count)
{
$temp_SIC = $this->summary_thread_container->stack_item_container_stack->first_stack_item_container;
$thread_count = 0;
while($temp_SIC)
{
$thread_count++;
echo "************************************** Thread Count: {$thread_count} \n";
$temp_SIC = $temp_SIC->Get_Next_Stack_Item_Container();
}
echo "*** END.\n";
if($this->thread_count == $thread_count)
echo "*** THREAD COUNTS MATCH.\n";
else
echo "*** THREAD COUNTS DO NOT MATCH.\n";
}
$this->Compile_Summary();
}
public function Compile_Summary(){ $this->summary_thread_container->Compile_Summary(); }
}
class Stack_Item_Container_Stack
{
public $first_stack_item_container;
private $thread_count;
private $summary_thread_container;
function __construct()
{
$this->first_stack_item_container = null;
$this->thread_count = 0;
for($i = 0; $i < 5; $i++)
{
echo " * Creating Stack Item Container: {$i}\n";
$this->thread_count++;
$this->Add_Stack_Item_Container(new Stack_Item_Container(rand(10, 20), $i, $this));
}
$this->summary_thread_container = new Summary_Thread_Container($this->thread_count, $this);
}
public function Add_Stack_Item_Container(Stack_Item_Container $stack_item_container)
{
echo " * Adding Stack Item Container *\n";
if($this->first_stack_item_container)
{
$temp_stack_item_container = $this->first_stack_item_container;
while($temp_stack_item_container->Get_Next_Stack_Item_Container())
$temp_stack_item_container = $temp_stack_item_container->Get_Next_Stack_Item_Container();
$temp_stack_item_container->Set_Next_Stack_Item_Container($stack_item_container);
}
else $this->first_stack_item_container = $stack_item_container;
}
public function Compile_Summary()
{
echo "\n";
echo "===============\n";
echo "=== Summary ===\n";
echo "===============\n";
echo "\n";
$temp_SIC = $this->first_stack_item_container;
while($temp_SIC)
{
echo " Thread ID {$temp_SIC->member_variables[0]} ({$temp_SIC->member_variables[4]}) has a Total of {$temp_SIC->member_variables[2]}";
echo "\n";
$temp_SIC = $temp_SIC->Get_Next_Stack_Item_Container();
}
echo "\n";
$this->Kill();
}
private function Kill()
{
while($this->first_stack_item_container)
{
$temp_SIC = $this->first_stack_item_container;
$this->first_stack_item_container = $this->first_stack_item_container->Get_Next_Stack_Item_Container();
$temp_SIC->Kill();
}
unset($this->summary_thread_container);
unset($this);
}
}
class Stack_Item_Container
{
private $stack_item_container_stack;
private $next_stack_item_container;
public $member_variables;
public $stack_item_thread;
function __construct($time, $index, Stack_Item_Container_Stack $stack_item_container_stack)
{
$this->stack_item_container_stack = $stack_item_container_stack;
$this->next_stack_item_container = null;
$this->member_variables = new Stackable();
$this->member_variables[] = -1;
$this->member_variables[] = $time;
$this->member_variables[] = 0;
$this->member_variables[] = false;
$this->member_variables[] = $index;
$this->stack_item_thread = new Stack_Item_Thread($this->member_variables, $this);
$this->stack_item_thread->start();
}
public function Get_Stack_Item_Container_Stack(){ return $this->stack_item_container_stack; }
public function Get_Next_Stack_Item_Container(){ return $this->next_stack_item_container; }
public function Set_Next_Stack_Item_Container(Stack_Item_Container $next_SIC){ $this->next_stack_item_container = $next_SIC; }
public function Kill()
{
$this->stack_item_thread->kill();
unset($this->member_variables);
unset($this);
}
}
class Stack_Item_Thread extends Worker
{
private $stack_item_container;
private $member_variables;
function __construct($member_variables, Stack_Item_Container $stack_item_container)
{
$this->member_variables = $member_variables;
$this->stack_item_container = $stack_item_container;
}
public function run()
{
$this->member_variables[0] = $this->getThreadId();
$total = 0;
echo "\n";
for($i = 0; $i < $this->member_variables[1]; $i++)
{
$total += $i;
$val = $i + 1;
echo "Thread ID ({$this->member_variables[4]}): {$this->member_variables[0]}:";
echo " Count: {$val} of {$this->member_variables[1]}";
echo "\n";
}
echo "\n";
$this->member_variables[2] = $total;
$this->member_variables[3] = true;
}
}
$stack_item_container_stack = new Stack_Item_Container_Stack();
OUTPUT 1 (When it does work):
************************************** Stack Thread Count: 5
*** START.
*** THREAD COUNTS DO NOT MATCH.
************************************** Thread Count: 1
************************************** Thread Count: 2
************************************** Thread Count: 3
************************************** Thread Count: 4
************************************** Thread Count: 5
*** END.
*** THREAD COUNTS MATCH.
===============
=== Summary ===
===============
Thread ID 139975400195840 (0) has a Total of 105
Thread ID 139975389705984 (1) has a Total of 153
Thread ID 139975378360064 (2) has a Total of 153
Thread ID 139975367014144 (3) has a Total of 55
Thread ID 139975130801920 (4) has a Total of 153
OUTPUT 2: (the first point that it will terminate):
************************************** Stack Thread Count: 5
*** START.
*** THREAD COUNTS DO NOT MATCH.
OUTPUT 3: (the second point that it will terminate)
************************************** Stack Thread Count: 5
*** START.
*** THREAD COUNTS DO NOT MATCH.
************************************** Thread Count: 1
************************************** Thread Count: 2
************************************** Thread Count: 3
************************************** Thread Count: 4
************************************** Thread Count: 5
*** END.
*** THREAD COUNTS MATCH.
Just to give you as much information as i can:
(might not be relevant but just in case)
Changes made to config:
File: /etc/sysctl.conf, Changes Made: net.ipv4.tcp_fin_timeout=10
File: php.ini, Changes Made: extension=php_pthreads.dll
Server:
Linux 2.6.32-504.8.1.el6.x86_64
PHP 5.5.13
Apache/2.2.15 (CentOS)
Max Requests Per Child: 4000 - Keep Alive: off - Max Per Connection: 100
Timeouts Connection: 60 - Keep-Alive: 15
Virtual Server No
Please help :) and questions would help ... anything i'm not seeing?
Thanks in advance
I cant test it myself, but if PHP exits normaly without errors its propably caused by the main thread exiting before any other thread could finish. Try Thread::join them (http://php.net/manual/en/thread.join.php) so the parent thread waits until they are finished.
Because Server costs are the greatest spending, we want to get more out of everyone.
How can we achieve, that more scripts can run on this server?
What the Scrips are doing:
We run 80 PHP Scripts on one Server and feed them with Jobs through Gearman.
The Scripts are looking up Websites with cURL, extract the needed Informations with Zend_Dom_Query and store the Data in an DB.
Each Script get feeded with ca. 1000 URLs which they have to look up.
Script example is down below.
What's the Server made of:
lshws outpu:
description: Computer
width: 64 bits
capabilities: vsyscall64 vsyscall32
*-core
description: Motherboard
physical id: 0
*-memory
description: System memory
physical id: 0
size: 8191GiB
*-cpu
product: Intel(R) Xeon(R) CPU E31230 # 3.20GHz
vendor: Intel Corp.
physical id: 1
bus info: cpu#0
width: 64 bits
capabilities: fpu fpu_exception wp vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp x86-64 constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave avx lahf_lm ida arat epb xsaveopt pln pts tpr_shadow vnmi flexpriority ept vpid
Nevertheless this is an V-Server it's the only V-Server running on that server. It has also not 8191GB Memory more like 16GB.
To show you how exhausted the server is, here's tops output:
top - 14:45:04 up 8 days, 3:10, 1 user, load average: 72.96, 72.51, 71.82
Tasks: 100 total, 72 running, 28 sleeping, 0 stopped, 0 zombie
Cpu(s): 87.5%us, 12.2%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.3%st
Mem: 8589934588k total, 4349016k used, 8585585572k free, 0k buffers
Swap: 0k total, 0k used, 0k free, 282516k cached
Not to forget here's the scripts main structure:
// Get the Infos on which to crawl on
$asin = explode(',', $job->workload());
try {
$userproducts = new App_Dbservices_U...();
$konkurrenz = new App_Dbservices_K...();
$repricingstream = new App_Dbservices_R...();
$err = 0;
for ($i = 0; $i < count($asin) - 3; $i = $i + 50) {
$mh = curl_multi_init();
$handles = array();
for ($j = $i; $j < $i + 50; $j++) {
if ((count($asin) - 3) > $j) {
if (isset($asin[$j])) {
// create a new single curl handle
$ch = curl_init();
// setting several options like url, timeout, returntransfer
// simulate multithreading by calling the wait.php scipt and sleeping for $rand seconds
$url = // URL
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 80);
// add this handle to the multi handle
$erroro[$j] = curl_errno($ch);
$errmsg[$j] = curl_error($ch);
curl_multi_add_handle($mh, $ch);
// put the handles in an array to loop this later on
$handles[] = $ch;
}
}
}
}
// execute the multi handle
$running = null;
do {
curl_multi_exec($mh, $running);
} while ($running > 0);
// get the content (if there is any)
$output = '';
for ($k = 0; $k < count($handles); $k++) {
// get the content of the handle
$output[$k] = curl_multi_getcontent($handles[$k]);
$_asin[$k]['asin'] = $asin[$j - 50 + $k];
$_asin[$k]['condition'] = $condition[$j - 50 + $k];
$_asin[$k]['pId'] = $pId[$j - 50 + $k];
if ($output[$k] != '')
{
// get the dom of each page
$dom = new Zend_Dom_Query($output[$k]);
// get the sellerInfos of each page
$seller = $dom->query('div.Offer');
if (count($seller) > 0) {
// get the price out of the string
$seller_i = 0;
$selfCameOver = false;
foreach ($seller as $d2) {
if ($seller_i <= 6 OR $selfCameOver === false) {
$itemHtml = '';
foreach($d2->childNodes as $node) {
$itemHtml .= $node->ownerDocument->saveHTML($node);
}
$dom = new Zend_Dom_Query($itemHtml);
$itemPrice = $dom->query('span.Price');
foreach($itemPrice as $ItemPrice)
{
$_asin[$k]['price_end'][$seller_i] = 0.00;
$_asin[$k]['shipping_end'][$seller_i] = 0.00;
if (preg_match('/[0-9]++(?>[,.][0-9]+)?+/', $ItemPrice->textContent, $rueckgabe)) {
$priceEnd = str_replace(',', '', str_replace('.', '', $rueckgabe[0][0]));
$priceLength = strlen($priceEnd);
$priceEnd = substr($priceEnd, 0, ($priceLength - 2)) . '.' . substr($priceEnd, ($priceLength - 2), 2);
$_asin[$k]['price_end'][$seller_i] = (float)$priceEnd;
}
}
}
$shippingPrice = $dom->query('span.ShippingPrice');
foreach($shippingPrice as $ShippingPrice)
{
preg_match_all('/[0-9]{1,}([\,\. ]?[0-9])*/', $ShippingPrice->textContent, $rueckgabe);
if (isset($rueckgabe[0][0])) {
// ...
}
}
$_asin[$k]['price_total_end'][$seller_i] = $_asin[$k]['price_end'][$seller_i] + $_asin[$k]['shipping_end'][$seller_i];
$conditionTag = $dom->query('.Condition');
foreach($conditionTag as $ConditionTag)
{
$_asin[$k]['main_con'][$seller_i]= 0;
$_asin[$k]['sub_con'][$seller_i] = 0;
$conditionValue = explode(' - ', $ConditionTag->textContent);
if(isset($conditionValue[0])){
// ...
}
if(isset($conditionValue[1])) {
// ...
}
}
$ratingItem = $dom->query('.Rating');
$_asin[$k]['bewertung_end'][$seller_i] = -1;
$_asin[$k]['stars_end'][$seller_i] = -1;
foreach($ratingItem as $RatingItem)
{
echo $RatingItem->textContent; // 99% positiv ... 12 Monaten ... 11.719 Bewertungen ...
// I want to get 99 (which is stars ) and 11719 (which is bewertungen )
preg_match_all('/[0-9]{1,}([\,\. ]?[0-9])*/', preg_replace('/,/', '.', $RatingItem->textContent), $rueckgabe);
if (isset($rueckgabe[0]) AND count($rueckgabe[0]) > 0) {
$_asin[$k]['bewertung_end'][$seller_i] = (int)str_replace('.', '', $rueckgabe[0][count($rueckgabe[0]) - 1]);
$_asin[$k]['stars_end'][$seller_i] = $rueckgabe[0][0];
}
}
$sellerType = $dom->query('.Name img');
$_asin[$k]['merchant_end'][$seller_i] = "N/A";
$_asin[$k]['name_end'][$seller_i] = "N/A";
$_asin[$k]['img_end'][$seller_i] = "N/A";
$_asin[$k]['konk_type'][$seller_i] = 'ERROR';
if(count($sellerType) == 1)
{
foreach($sellerType as $SellerType)
{
$imgAltText = $SellerType->getAttribute('alt');
$a = explode('.', $imgAltText);
// ...
}
}
elseif(count($sellerType) == 0)
{
$_asin[$k]['img_end'][$seller_i] = 'NO_IMG';
$_asin[$k]['konk_type'][$seller_i] = 'WO_IMG';
$sellerName = $dom->query('.Name b');
foreach($sellerName as $SellerName)
{
$_asin[$k]['name_end'][$seller_i] = $SellerName->textContent;
}
$sellerMerchant = $dom->query('.Name a');
foreach($sellerMerchant as $SellerMerchant)
{
$_asin[$k]['merchant_end'][$seller_i] = str_replace('=', '', substr($SellerMerchant->getAttribute('href'), -14));
}
}
unset($rueckgabe);
}
$seller_i++;
}
}
}
// remove the handle from the multi handle
curl_multi_remove_handle($mh, $handles[$k]);
}
// Update Price ...
// Update Shipping ...
// Update Conc ...
unset($_asin);
// close the multi curl handle to free system resources
curl_multi_close($mh);
}
} catch (Exception $e) {
$error = new Repricing_Dbservices_Error();
$error->setError($id, $e->getMessage(), $e->getLine(), $e->getFile());
}
And also the script for the price update ( the other update-statements looks similiar)
$this->db->beginTransaction();
try {
for ($i = 0; $i < count($asin); $i++) {
if (isset($asin[$i]['price_total_end'])) {
if (count($asin[$i]['price_total_end']) > 1) {
if ($asin[$i]['price_total_end'][0] > 0) {
$this->db->query("UPDATE u... SET lowest_price = ? , last_lowest_price_update = ? WHERE id = ?", array(
$asin[$i]['price_total_end'][1],
date("Y-m-d H:i:s", time()),
$asin[$i]['pId']
));
}
} elseif (count($asin[$i]['price_total_end']) == 1) {
if ($asin[$i]['price_total_end'][0] >= 0) {
$this->db->query("UPDATE u... SET lowest_price = ? , last_lowest_price_update = ? WHERE id = ?", array(
-1,
date("Y-m-d H:i:s", time()),
$asin[$i]['pId']
));
}
}
}
}
$this->db->commit();
} catch (Exception $e) {
$this->db->rollBack();
echo $e->getMessage();
}
$this->db->closeConnection();
Do we have an big performance leak in our script, should we go along with an other language, or any other techniques? Every suggestion is highly appreciated.
You can replace all these kind of lines:
preg_match_all('/[0-9]{1,}([\,\. ]?[0-9])*/', $ItemPrice->textContent, $rueckgabe);
if (isset($rueckgabe[0])) {
// ...
}
by:
if (preg_match('/([0-9]++)(?>[.,]([0-9]++))?+/', $ItemPrice->textContent, $rueckgabe)) {
unset($rueckgabe[0]);
$priceEnd = sprintf("%01.2f", implode('.', $rueckgabe));
$_asin[$k]['price_end'][$seller_i] = $priceEnd;
}
You should replace all your for loops with foreach (then you avoid the count on each loop as RaymondN notices it). Example:
Instead of:
for ($k = 0; $k < count($handles); $k++) {
you write:
foreach($handles as $k=>$handle) {
// you can replace $handles[$k] by $handle
It is not very useful to convert the current datetime and format it in "Y-m-d H:i:s" since you can do directly the same with the mySQL statement NOW().
DonĀ“t use count functions in the for loops that would save some CPU cycles ..
but use.
$m = count($array) - 1;
for ($i = 0 ; $i < $m ; $i++) {}
Whats the PHP version a new PHP version could possible perform much better.
The most notable thing here is that you are sharding the data too much - when your load average is higher than the number of CPUs, then the OS will start pre-emppting jobs instead of waiting for them to yeild the CPU. As a result overall throughput drops significantly. You seem to have a single CPU - for a cpu bound system like this running on a single core, I'd try 2, 4, 8 and 16 processes to see which gives the optimal behaviour (assuming you need to make the code fit the hardware rather than the other way around).
Your next problem is that the zend framework is very cpu and memory hungry: Zend is 4 times slower than unadorned PHP.
You've got a lot of inlined code in the loops here - while inlining does help performance it makes it a lot more difficult to get useful data out of a profiler - hence my next step after making the Zend-free and reducing the concurrency would be to structure the code into functions and profile it.
I have the problem that in a PHP application Gearman jobs sometimes are passed to more than one worker. I could reduce a code to reproduce it into one file. Now I am not sure if this is a bug in Gearman or a bug in the pecl library or maybe in my code.
Here is the code to reproduce the error:
#!/usr/bin/php
<?php
// Try 'standard', 'exception' or 'exception-sleep'.
$sWorker = 'exception';
// Detect run mode "client" or "worker".
if (!isset($argv[1]))
$sMode = 'client';
else
$sMode = 'worker-' . $sWorker;
$sLogFilePath = __DIR__ . '/log.txt';
switch ($sMode) {
case 'client':
// Remove all queued test jobs and quit if there are test workers running.
prepare();
// Init the greaman client.
$Client= new GearmanClient;
$Client->addServer();
// Empty the log file.
file_put_contents($sLogFilePath, '');
// Start some worker processes.
$aPids = array();
for ($i = 0; $i < 100; $i++)
$aPids[] = exec('php ' . __FILE__ . ' worker > /dev/null 2>&1 & echo $!');
// Start some jobs. Also try doHigh(), doBackground() and
// doBackgroundHigh();
for ($i = 0; $i < 50; $i++)
$Client->doNormal('test', $i);
// Wait a second (when running jobs in background).
// sleep(1);
// Prepare the log file entries.
$aJobs = array();
$aLines = file($sLogFilePath);
foreach ($aLines as $sLine) {
list($sTime, $sPid, $sHandle, $sWorkload) = $aAttributes = explode("\t", $sLine);
$sWorkload = trim($sWorkload);
if (!isset($aJobs[$sWorkload]))
$aJobs[$sWorkload] = array();
$aJobs[$sWorkload][] = $aAttributes;
}
// Remove all jobs that have been passed to only one worker as expected.
foreach ($aJobs as $sWorkload => $aJob) {
if (count($aJob) === 1)
unset($aJobs[$sWorkload]);
}
echo "\n\n";
if (empty($aJobs))
echo "No job has been passed to more than one worker.";
else {
echo "Those jobs has been passed more than one times to a worker:\n";
foreach ($aJobs as $sWorload => $aJob) {
echo "\nJob #" . $sWorload . ":\n";
foreach ($aJob as $aAttributes)
echo " $aAttributes[2] (Worker PID: $aAttributes[1])\n";
}
}
echo "\n\n";
// Kill all started workers.
foreach ($aPids as $sPid)
exec('kill ' . $sPid . ' > /dev/null 2>&1');
break;
case 'worker-standard':
$Worker = new GearmanWorker;
$Worker->addServer();
$Worker->addFunction('test', 'logJob');
$bShutdown = false;
while ($Worker->work())
if ($bShutdown)
continue;
break;
case 'worker-exception':
try {
$Worker = new GearmanWorker;
$Worker->addServer();
$Worker->addFunction('test', 'logJob');
$bShutdown = false;
while ($Worker->work())
if ($bShutdown)
throw new \Exception;
} catch (\Exception $E) {
}
break;
case 'worker-exception-sleep':
try {
$Worker = new GearmanWorker;
$Worker->addServer();
$Worker->addFunction('test', 'logJob');
$bShutdown = false;
while ($Worker->work())
{
if ($bShutdown) {
sleep(1);
throw new \Exception;
}
}
} catch (\Exception $E) {
}
break;
}
function logJob(\GearmanJob $Job)
{
global $bShutdown, $sLogFilePath;
$sLine = microtime(true) . "\t" . getmypid() . "\t" . $Job->handle() . "\t" . $Job->workload() . "\n";
file_put_contents($sLogFilePath, $sLine, FILE_APPEND);
$bShutdown = true;
}
function prepare()
{
$rGearman = fsockopen('127.0.0.1', '4730', $iErrno, $sErrstr, 3);
$aBuffer = array();
fputs ($rGearman, 'STATUS' . PHP_EOL);
stream_set_timeout($rGearman, 1);
while (!feof($rGearman))
if ('.' . PHP_EOL !== $sLine = fgets($rGearman, 128))
$aBuffer[] = $sLine;
else
break;
fclose($rGearman);
$bJobsInQueue = false;
$bWorkersRunning = false;
foreach ($aBuffer as $sFunctionLine) {
list($sFunctionName, $iQueuedJobs, $iRunningJobs, $iWorkers) = explode("\t", $sFunctionLine);
if ('test' === $sFunctionName) {
if (0 != $iQueuedJobs)
$bJobsInQueue = true;
if (0 != $iWorkers)
$bWorkersRunning = true;;
}
}
// Exit if there are workers running.
if ($bWorkersRunning)
die("There are some Gearman workers running that have registered a 'test' function. Please stop these workers and run again.\n\n");
// If there are test jobs in the queue start a worker that eat up the jobs.
if ($bJobsInQueue) {
$sPid = exec('gearman -n -w -f test > /dev/null 2>&1 & echo $!');
sleep(1);
exec ("kill $sPid > /dev/null 2>&1");
// Repeat this method to make sure all jobs are removed.
prepare();
}
}
When you run this code on the command line it should output "No job
has been passed to more than one worker." but insted it alway outputs a list of some jobs that have been passed to more than one worker. The error doesn't appear if you set $sWorker = 'standard'; or 'exception-sleep'.
It would help me a lot if you could run the code and tell me if you are able to reproduce the error of if I have a bug in the code.
Had exactly the same issue with Gearman 0.24, PECL lib 1.0.2. Was able to reproduce the error with your script every time.
An older version of Gearman (0.14 I think) used to work fine.
Upgrading Gearman to 0.33 fixed the issue.
I have a PHP page I need to limit execution access of
to only clients inside our firewall.
How would I write a php-script that can look up the clients
ip-address and match it to a ip-range (for instance 10...* or 200.10.10.*).
You can use ip2long to convert dotted quads to long values, then just perform some arithmetic to check a given network/mask combination:
$network=ip2long("200.10.10.0");
$mask=ip2long("255.255.255.0");
$remote=ip2long($_SERVER['REMOTE_ADDR']);
if (($remote & $mask) == $network)
{
//match!
}
else
{
//does not match!
}
Well, assuming you're using Apache, there's a module called mod_authz_host that you can use.
Together with the file directive, you could limit access to a given php script for a range of ip addresses.
Here is the link to the documentation on that module:
http://httpd.apache.org/docs/2.2/mod/mod_authz_host.html
Here's a quick example (assuming your php file is called admin.php):
<file admin.php>
Order Deny,Allow
Deny from all
Allow from 200.10.10
</file>
The added benefit to the other solution suggested here is that you can control the security aspects from outside your application logic - a more flexible approach that does not impose any limitations on your PHP code.
PHP Limit/Block Website requests for Spiders/Bots/Clients etc.
Here i have written a PHP function which can Block unwanted Requests to reduce your Website-Traffic. God for Spiders, Bots and annoying Clients.
DEMO:
http://szczepan.info/9-webdesign/php/1-php-limit-block-website-requests-for-spiders-bots-clients-etc.html
/* Function which can Block unwanted Requests
* #return boolean/array status
*/
function requestBlocker()
{
/*
Version 1.0 11 Jan 2013
Author: Szczepan K
http://www.szczepan.info
me[#] szczepan [dot] info
###Description###
A PHP function which can Block unwanted Requests to reduce your Website-Traffic.
God for Spiders, Bots and annoying Clients.
*/
$dir = 'requestBlocker/'; ## Create & set directory writeable!!!!
$rules = array(
#You can add multiple Rules in a array like this one here
#Notice that large "sec definitions" (like 60*60*60) will blow up your client File
array(
//if >5 requests in 5 Seconds then Block client 15 Seconds
'requests' => 5, //5 requests
'sek' => 5, //5 requests in 5 Seconds
'blockTime' => 15 // Block client 15 Seconds
),
array(
//if >10 requests in 30 Seconds then Block client 20 Seconds
'requests' => 10, //10 requests
'sek' => 30, //10 requests in 30 Seconds
'blockTime' => 20 // Block client 20 Seconds
),
array(
//if >200 requests in 1 Hour then Block client 10 Minutes
'requests' => 200, //200 requests
'sek' => 60 * 60, //200 requests in 1 Hour
'blockTime' => 60 * 10 // Block client 10 Minutes
)
);
$time = time();
$blockIt = array();
$user = array();
#Set Unique Name for each Client-File
$user[] = isset($_SERVER['REMOTE_ADDR']) ? $_SERVER['REMOTE_ADDR'] : 'IP_unknown';
$user[] = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';
$user[] = strtolower(gethostbyaddr($user[0]));
# Notice that i use files because bots does not accept Sessions
$botFile = $dir . substr($user[0], 0, 8) . '_' . substr(md5(join('', $user)), 0, 5) . '.txt';
if (file_exists($botFile)) {
$file = file_get_contents($botFile);
$client = unserialize($file);
} else {
$client = array();
$client['time'][$time] = 0;
}
# Set/Unset Blocktime for blocked Clients
if (isset($client['block'])) {
foreach ($client['block'] as $ruleNr => $timestampPast) {
$left = $time - $timestampPast;
if (($left) > $rules[$ruleNr]['blockTime']) {
unset($client['block'][$ruleNr]);
continue;
}
$blockIt[] = 'Block active for Rule: ' . $ruleNr . ' - unlock in ' . ($left - $rules[$ruleNr]['blockTime']) . ' Sec.';
}
if (!empty($blockIt)) {
return $blockIt;
}
}
# log/count each access
if (!isset($client['time'][$time])) {
$client['time'][$time] = 1;
} else {
$client['time'][$time]++;
}
#check the Rules for Client
$min = array(
0
);
foreach ($rules as $ruleNr => $v) {
$i = 0;
$tr = false;
$sum[$ruleNr] = '';
$requests = $v['requests'];
$sek = $v['sek'];
foreach ($client['time'] as $timestampPast => $count) {
if (($time - $timestampPast) < $sek) {
$sum[$ruleNr] += $count;
if ($tr == false) {
#register non-use Timestamps for File
$min[] = $i;
unset($min[0]);
$tr = true;
}
}
$i++;
}
if ($sum[$ruleNr] > $requests) {
$blockIt[] = 'Limit : ' . $ruleNr . '=' . $requests . ' requests in ' . $sek . ' seconds!';
$client['block'][$ruleNr] = $time;
}
}
$min = min($min) - 1;
#drop non-use Timestamps in File
foreach ($client['time'] as $k => $v) {
if (!($min <= $i)) {
unset($client['time'][$k]);
}
}
$file = file_put_contents($botFile, serialize($client));
return $blockIt;
}
if ($t = requestBlocker()) {
echo 'dont pass here!';
print_R($t);
} else {
echo "go on!";
}