summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorUrban Müller2020-04-16 20:57:01 +0200
committerUrban Müller2020-04-16 20:57:01 +0200
commitf3ae6e3c50d217d3fd46df8edf8079547e6f2864 (patch)
tree18b32a7ae2f5640c9d14f0f0f30207460a51dba3
parent7408797b1716210ff7e1c9f18629b0e65a70f1c9 (diff)
downloaditools-f3ae6e3c50d217d3fd46df8edf8079547e6f2864.tar.gz
itools-f3ae6e3c50d217d3fd46df8edf8079547e6f2864.tar.bz2
itools-f3ae6e3c50d217d3fd46df8edf8079547e6f2864.zip
allow postprocessors like json_decode
-rw-r--r--it_url.class8
1 files changed, 7 insertions, 1 deletions
diff --git a/it_url.class b/it_url.class
index a08237a..8eb7aa8 100644
--- a/it_url.class
+++ b/it_url.class
@@ -98,6 +98,7 @@ function is_reachable($timeout = 5)
* @param $p['retries'] Number of retries if download fails, default 1
* @param $p['retrysleep'] Number of seconds to wait before retry (additional to fetchsleep), fractions ok
* @param $p['compression'] use compression (uses curl to do that)
+ * @param $p['postprocess'] function called with content and $p which has it_error. returns content or null
* @return contents of resulting page, considering redirects, excluding headers, or false on error
*/
function get($p=null, $timeout=5)
@@ -121,6 +122,8 @@ function get($p=null, $timeout=5)
$url = new it_url($p['url']);
$result = $url->request($p + ['followlocation' => true]);
+ if ($p['postprocess'])
+ $result = $p['postprocess']($result, ['it_error' => $p['retries'] > 0 ? false : ['title' => "invalid content from " . $p['url']]]);
if (!$result && $p['retries'] > 0 && !it::match('^(4..|204)$', $url->result))
{
@@ -383,9 +386,11 @@ static function get_multi($p=null)
{
$key = $keys[$info['handle']];
$content = curl_multi_getcontent($info['handle']);
+ if (isset($p['postprocess']))
+ $content = $p['postprocess']($content, ['it_error' => $retries[$key] < $p['retries'] ? false : ['title' => "invalid content from " . $urls[$key]]]);
EDC('reqtimings', $key, $info['result'], (gettimeofday(true) - $start) * 1000);
- if ($info['result'] == CURLE_OK) {
+ if ($info['result'] == CURLE_OK && $content !== null) {
if (!$p['noresults'])
$results_unordered[$key] = $content;
@@ -397,6 +402,7 @@ static function get_multi($p=null)
} else if($retries[$key]++ < $p['retries']) {
$closehandle($key); # closehandle must be called before addhandle as we use the same key
$addhandle($key, $urls[$key]);
+ $mrc = CURLM_CALL_MULTI_PERFORM; # force continue if this was last handle
} else {
$results_unordered[$key] = false;
unset($urls[$key]);