From 2946cad416b247327c033f61fda8270c6771248b Mon Sep 17 00:00:00 2001
From: Urban Müller
Date: Thu, 18 Apr 2024 18:17:42 +0200
Subject: introduce empty_on_fail, get keepfailed to work

---
 it_url.class | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

(limited to 'it_url.class')

diff --git a/it_url.class b/it_url.class
index 3d407eb..da1de16 100644
--- a/it_url.class
+++ b/it_url.class
@@ -109,22 +109,20 @@ static function _postprocess($data, $p)
  * @param $p['accept_encoding'] Contents of the "Accept-Encoding: " header. Enables decoding of the response. Set to null to disable, "" (default) for all supported encodings.
  * @param $p['postprocess']   function called with content and $p which has it_error. returns content or null (which triggers retry)
  * @param $p['protocols']     Array of protocols to accept, defaults to ['http', 'https'], @see curl_opts for other values
+ * @param $p['empty_on_fail'] Return empty page if http status code is >= 400
  * @return Content of resulting page (considering redirects, excluding headers or false on error) or array (empty on error) if 'assoc' => true
  */
-static function get($p = [], $timeout = null)
+static function get($p = [])
 {
-	return (new static)->_get($p, $timeout);
+	return (new static)->_get($p);
 }
 
 /**
  * Non-static alias for get so we can make get() static
  */
-function _get($p = [], $timeout = null)
+function _get($p = [])
 {
-	if (isset($timeout))
-		it::error("Deprecated second argument of it_url::get()!");
-	if (is_string($p))
-		$p = array('url' => $p, 'timeout' => 5);
+	$p = is_string($p) ? ['url' => $p, 'timeout' => 5] : $p;
 	$p += array('retries' => 1);
 
 	if (($filter = EDC('req')) && ($filter == 1 || strstr($p['url'], "/$filter.")))
@@ -139,6 +137,7 @@ function _get($p = [], $timeout = null)
 	$result = $this->request($p + ['followlocation' => true]);
 	$result = self::_postprocess($result, $p);
 
+	# FIXME 2024-07 UM some failures never send errs in request() because retries > 0
 	if ($p['retries'] > 0 && ((!$result && !it::match('^(204|4..)$', $this->result)) || it::match(self::$retryable, $this->result)))
 	{
 		usleep($p['retrysleep']*1000000);
@@ -353,6 +352,8 @@ function request($p=array())
 		}
 		else
 		{
+			if ($url->result >= 400 && ($p['empty_on_fail'] || $p['keepfailed']))
+				$got = $url->data = false;
 			$result =& $url->data;
 			$this->errstr = "HTTP Status " . $url->result;
 		}
@@ -576,7 +577,7 @@ static function get_cache_filename($p)
  * @param $p['preprocess']  callback function (or array for methods) to change received file or array('function' => ..., 'in' => $src, 'out' => $dst, ...) with callback function plus args
  * @param $p['safety']      DEPRECATED. see $p['it_error']
  * @param $p['it_error']    parameters for it::error(), false means ignore errors, anything else gets passed to it::error() if errors occur
- * @param $p['keepfailed']  keep old versions of files if download fails (sending alerts conservatively)
+ * @param $p['keepfailed']  keep old versions of files if download fails
  * @param $p['returnheaders'] Return array($path, $headers) instead of simply $path
  * @param $p['postprocess'] UNSUPPORTED, use ::get_cache_contents
  * @param $p['lock']        prevent multiple requests to same url from different processes [true]
@@ -740,7 +741,7 @@ static function _expired($path, $maxage, $randomexpire = 0)
 {
 	if ($result = EDC('nocache') ? false : @filemtime($path))
 	{
-		if (time() - $result > $maxage || rand(0, 100000) <= $randomexpire * 100000)
+		if (time() - $result >= $maxage || rand(0, 100000) <= $randomexpire * 100000)
 			EDC('getcache', "expired", $maxage, $path);
 		else
 			$result = false;
-- 
cgit v1.2.3