correctly retry in ::get_multi, do retry in status 5xx in ::get

3 files changed, 33 insertions, 5 deletions

diff --git a/it_url.class b/it_url.class
index 318168c..51cf9fb 100644
--- a/it_url.class
+++ b/it_url.class
@@ -136,7 +136,7 @@ function _get($p = [], $timeout = null)
 	$result = $this->request($p + ['followlocation' => true]);
 	$result = self::_postprocess($result, $p);
 
-	if (!$result && $p['retries'] > 0 && !it::match('^(4..|204)$', $this->result))
+	if ($p['retries'] > 0 && ((!$result && !it::match('^(204|4..)$', $this->result)) || it::match('^(5..)$', $this->result)))
 	{
 		usleep($p['retrysleep']*1000000);
 		$result = $this->_get(array('retries' => $p['retries'] - 1) + $p);
@@ -393,10 +393,11 @@ static function get_multi($p=null)
 	} while ($mrc == CURLM_CALL_MULTI_PERFORM);
 
 	$timeout = 0.001;	# Very short timeout to work around problem with first select call on cURL 7.25.0
-	while (!$abort && (($active && $mrc == CURLM_OK) || count($handles) > 0))
+	while (!$abort && (($active && $mrc == CURLM_OK) || count($handles) > 0 || $sleepuntils))
 	{
 		if (curl_multi_select($mh, $timeout) == -1)
 			usleep($timeout * 1000000);
+
 		do {
 			$mrc = curl_multi_exec($mh, $active);
 
@@ -417,14 +418,14 @@ static function get_multi($p=null)
 						if (is_array($urls[$key]) && ($handler = $urls[$key]['handler']))
 							$abort = $handler($info['handle'], $content);
 
-						unset($urls[$key]);
+						if (!it::match('^(2..|4..)$', curl_getinfo($handles[$key], CURLINFO_RESPONSE_CODE)) && $retries[$key]++ < $p['retries'])
+							$sleepuntils[$key] = microtime(true) + $p['retrysleep'];
 						$closehandle($key);
 					} else if($retries[$key]++ < $p['retries']) {
 						$closehandle($key);  # closehandle must be called before addhandle as we use the same key
 						$sleepuntils[$key] = microtime(true) + $p['retrysleep'];
 					} else {
 						$results_unordered[$key] = false;
-						unset($urls[$key]);
 						$closehandle($key);
 					}
 
diff --git a/test/it_url.t b/test/it_url.t
index 5bac758..1636bd6 100755
--- a/test/it_url.t
+++ b/test/it_url.t
@@ -316,6 +316,27 @@ ok(strpos($pages['a'], '</html>'), 'it_url::get_multi got first url'); # UTF8SAF
 ok(strpos($pages['b'], '</html>'), 'it_url::get_multi got second url'); # UTF8SAFE
 is(count($pages), 2, 'it_url::get_multi no additional array elements');
 
+handle_server(
+	is(
+		it_url::get('http://localhost:8000/maybe_error?chance=0'),
+		"success"
+	),
+	is(
+		it_url::get('http://localhost:8000/maybe_error?chance=100'),
+		"failure"
+	),
+	is(
+		it_url::get('url' => 'http://localhost:8000/maybe_error?chance=10', 'retries' => 10),
+		"success",
+		"Retry on sporadically failing url in ::get"
+	),
+	is(
+		it_url::get_multi('urls' => ['http://localhost:8000/maybe_error?chance=10'], 'retries' => 10),
+		["success"],
+		"Retry on sporadically failing url in ::get_multi"
+	),
+);
+
 is(it_url::parse("/foo"), ["/foo"], "it_url::parse path only");
 is(it_url::parse("/foo?"), ["/foo"], "it_url::parse empty parameter");
 is(it_url::parse("/foo?bar=baz&qux=quux"), ["/foo", 'bar' => "baz", 'qux' => "quux"], "it_url::parse parameters");
diff --git a/test/it_url.testserver.php b/test/it_url.testserver.php
index 64e6674..f895d61 100644
--- a/test/it_url.testserver.php
+++ b/test/it_url.testserver.php
@@ -1,5 +1,5 @@
 <?php
-$stderr = fopen('php://stderr', 'w');
+$stderr = it::fopen('php://stderr', 'w');
 fwrite($stderr, "Got Request: '" . $_SERVER['REQUEST_URI'] . "'\n");
 $base = 'http://' . $_SERVER['HTTP_HOST'];
 
@@ -69,6 +69,12 @@ switch ($_SERVER['PHP_SELF'])
 		http_response_code(204);
 		break;
 
+	case "/maybe_error":
+		$iserror = rand(0, 99) <= $_REQUEST['chance'];
+		http_response_code($iserror ? 500 : 200);
+		echo $iserror ? "failure" : "success";
+		break;
+
 	default:
 		http_response_code(404);
 		fwrite($stderr, "Unknown path '$_SERVER[PHP_SELF]' not handled!\n");