Run 10 Curl threads then wait till finished

[imperium2335]

Hi,

 

I have a system where I want to process a number of urls at the same time, so I'm using Curl multi to do this.

 

What I would like to know is how to set it to only do 10 urls at a time, at any one time, so curl doesn't end up creating 100s of connections because I may have several hundred urls needing to be processed.

 

Does anyone have any idea how this can be done? Below is the code that opens 100s of connections which is what I don't want, I just need to control it a little. Thanks in advance.

 

$multithread = curl_multi_init() ;

foreach($urls as $key => $currenturl) {

${'thread' . $key} = "THREAD " . $key ; // Create a variable called $threadN, where N is the current array key, i.e. 0 for the first array item, then 1 etc.
if($diags == TRUE)echo "Attempting to create ${'thread' . $key}.<br />" ;
$log = "Initialising CURL thread: ${'thread' . $key}.\n" ; fwrite($fh, $log) ;
${'thread' . $key} = curl_init() ; 
curl_setopt(${'thread' . $key}, CURLOPT_URL, $searchurls) ;
curl_setopt(${'thread' . $key}, CURLOPT_HEADER, 0) ;
curl_setopt(${'thread' . $key}, CURLOPT_RETURNTRANSFER, 1); // Send data to browser or string(1).
curl_setopt(${'thread' . $key}, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt(${'thread' . $key}, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt(${'thread' . $key}, CURLOPT_MAXREDIRS, 3);
curl_setopt(${'thread' . $key}, CURLOPT_LOW_SPEED_LIMIT, 5120);
curl_setopt(${'thread' . $key}, CURLOPT_LOW_SPEED_TIME, 3);
curl_setopt(${'thread' . $key}, CURLOPT_COOKIEJAR, "cookies.txt");
$A = rand(0, 19) ;
curl_setopt(${'thread' . $key}, CURLOPT_REFERER, $referers[$A]);
if($diags == TRUE)echo "Pretending to come from $referers[$A]<br />" ;
$A = rand(0, 17) ;
curl_setopt(${'thread' . $key}, CURLOPT_USERAGENT, $agents[$A]); //Randomly assign a browser agent.
if($diags == TRUE)echo "Masquerading as the $agents[$A] browser.<br />" ;
if(curl_multi_add_handle($multithread,${'thread' . $key}) != 0) {
	echo "Critical error: Can not assign thread ${'thread' . $key} to multi." ;
	$log = "Initialisation error. Thread could not be assigned.\n" ; fwrite($fh, $log) ;
}
else $log = "Initialisation complete.\n" ; fwrite($fh, $log) ;

}
$active = NULL ;

do {
curl_multi_exec($multithread, $active);

} while($active > 0);
$log = "Thread execution complete.\n" ; fwrite($fh, $log) ;

foreach($tstack as $key => $searchurls) {

$rawbuild = curl_multi_getcontent(${'thread' . $key}) ;
array_push($finals, $rawbuild) ;

}