From 3538e258590fda78b697ef0e8280585f818034e4 Mon Sep 17 00:00:00 2001
From: Urban Müller
Date: Tue, 4 Aug 2020 17:55:30 +0200
Subject: correctly retry in ::get_multi, do retry in status 5xx in ::get

---
 it_url.class | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'it_url.class')

diff --git a/it_url.class b/it_url.class
index 318168c..51cf9fb 100644
--- a/it_url.class
+++ b/it_url.class
@@ -136,7 +136,7 @@ function _get($p = [], $timeout = null)
 	$result = $this->request($p + ['followlocation' => true]);
 	$result = self::_postprocess($result, $p);
 
-	if (!$result && $p['retries'] > 0 && !it::match('^(4..|204)$', $this->result))
+	if ($p['retries'] > 0 && ((!$result && !it::match('^(204|4..)$', $this->result)) || it::match('^(5..)$', $this->result)))
 	{
 		usleep($p['retrysleep']*1000000);
 		$result = $this->_get(array('retries' => $p['retries'] - 1) + $p);
@@ -393,10 +393,11 @@ static function get_multi($p=null)
 	} while ($mrc == CURLM_CALL_MULTI_PERFORM);
 
 	$timeout = 0.001;	# Very short timeout to work around problem with first select call on cURL 7.25.0
-	while (!$abort && (($active && $mrc == CURLM_OK) || count($handles) > 0))
+	while (!$abort && (($active && $mrc == CURLM_OK) || count($handles) > 0 || $sleepuntils))
 	{
 		if (curl_multi_select($mh, $timeout) == -1)
 			usleep($timeout * 1000000);
+
 		do {
 			$mrc = curl_multi_exec($mh, $active);
 
@@ -417,14 +418,14 @@ static function get_multi($p=null)
 						if (is_array($urls[$key]) && ($handler = $urls[$key]['handler']))
 							$abort = $handler($info['handle'], $content);
 
-						unset($urls[$key]);
+						if (!it::match('^(2..|4..)$', curl_getinfo($handles[$key], CURLINFO_RESPONSE_CODE)) && $retries[$key]++ < $p['retries'])
+							$sleepuntils[$key] = microtime(true) + $p['retrysleep'];
 						$closehandle($key);
 					} else if($retries[$key]++ < $p['retries']) {
 						$closehandle($key);  # closehandle must be called before addhandle as we use the same key
 						$sleepuntils[$key] = microtime(true) + $p['retrysleep'];
 					} else {
 						$results_unordered[$key] = false;
-						unset($urls[$key]);
 						$closehandle($key);
 					}
 
-- 
cgit v1.2.3