implement retries for get_multi

1 files changed, 28 insertions, 11 deletions

diff --git a/it_url.class b/it_url.class
index 53bf949..85162db 100644
--- a/it_url.class
+++ b/it_url.class
@@ -295,14 +295,26 @@ function get_multi($p=null)
 	foreach ($p['urls'] as $key => $url)
 		$urls[$key] = is_array($url) ? $url : array('url' => $url);
 
-	foreach ($urls as $key => $url)
-	{
+	$keys = $handles = $retries = [];
+	$addhandle = function ($key) use (&$keys, &$handles, $urls, $opts, $mh) {
 		$handle = curl_init();
-		curl_setopt($handle, CURLOPT_URL, it::replace([ '^//' => "http://" ], $url['url']));
+		curl_setopt($handle, CURLOPT_URL, it::replace([ '^//' => "http://" ], $urls[$key]['url']));
 		curl_setopt_array($handle, $opts);
 		curl_multi_add_handle($mh, $handle);
 		$keys[$handle] = $key;
 		$handles[$key] = $handle;
+	};
+	$closehandle = function ($key) use (&$keys, &$handles, $mh) {
+		curl_multi_remove_handle($mh, $handles[$key]);
+		curl_close($handles[$key]);
+		unset($keys[$handles[$key]]);
+		unset($handles[$key]);
+	};
+
+	foreach ($urls as $key => $dummy)
+	{
+		$addhandle($key);
+		$retries[$key] = 0;
 	}
 
 	$start = microtime(true);
@@ -326,27 +338,32 @@ function get_multi($p=null)
 				if ($info['msg'] == CURLMSG_DONE)
 				{
 					$key = $keys[$info['handle']];
+					$content = curl_multi_getcontent($info['handle']);
+
+					$closehandle($key);
+
 					EDC('reqtimings', $key, $info['result'], (microtime(true) - $start) * 1000);
 					if ($info['result'] == CURLE_OK)
-						$results_unordered[$key] = curl_multi_getcontent($info['handle']);
+						$results_unordered[$key] = $content;
+					else if($retries[$key]++ < $p['retries'])
+						$addhandle($key);
 					else
 						$results_unordered[$key] = false;
+
 					if (($handler = $urls[$keys[$info['handle']]]['handler']))
 						$abort = $handler($info['result'], $results_unordered[$key]);
+
 				}
 			}
 		} while ($mrc == CURLM_CALL_MULTI_PERFORM);
 		$timeout = 0.1;	# Longer delay to avoid busy loop but shorter than default of 1s in case we stil hit cURL 7.25.0 problem
 	}
 
-	$results = array();
-	foreach ($handles as $key => $handle) {
-		curl_multi_remove_handle($mh, $handle);
-		curl_close($handle);
-		$results[$key] = $results_unordered[$key];
-	}
+	foreach ($handles as $key => $dummy)
+		$closehandle($key);
 	curl_multi_close($mh);
-	return $results;
+
+	return it::filter_keys($results_unordered, array_keys($urls), ['reorder' => true]);
 }
 
 /**