diff options
| -rw-r--r-- | it.class | 25 | ||||
| -rw-r--r-- | it_dbi.class | 15 | ||||
| -rw-r--r-- | it_debug.class | 3 | ||||
| -rw-r--r-- | it_mail.class | 11 | ||||
| -rw-r--r-- | it_url.class | 44 | ||||
| -rw-r--r-- | it_xml.class | 1 | ||||
| -rwxr-xr-x | test/it_dbi.t | 2 | ||||
| -rwxr-xr-x | test/it_html.t | 2 | ||||
| -rw-r--r-- | test/it_url.testserver.php | 4 | ||||
| -rwxr-xr-x | test/it_url_slow.t | 7 |
10 files changed, 71 insertions, 43 deletions
@@ -270,7 +270,7 @@ static function error($p = array(), $extra = null) if (strlen($p['body']) > 500000 || it::match('[\x00-\x08\x0E-\x1F]', $p['body'], ['utf8' => false]) !== null) { @file_put_contents($datafn = "$home/tmp/error-" . substr(md5($p['body']), 0, 2), $p['body']); # NOPHPLINT - $p['body'] = " See " . getenv('HOSTNAME') . ":$datafn"; + $p['body'] = "see " . getenv('HOSTNAME') . ":$datafn"; } $service = it::match('[^.]*', $GLOBALS['ULTRASITE']); @@ -1021,6 +1021,22 @@ static function _stdin_next() } /** + * Get all lines from command line arguments or stdin. + * Transparently reads existing files in the command line arguments and passes on all other arguments. + * Note: You need to call getopt() before using this function. + */ +static function gets_all() +{ + while (it::_stdin_next()) { + if (($fd = $GLOBALS['it_stdin']['fd'])) + while (($line = fgets($fd)) !== false) + yield rtrim($line, "\n"); + else + yield $GLOBALS['it_stdin']['filename']; + } +} + +/** * Get one line from stdin (or files given on command line) a la Perl <>. * Note: You need to call getopt() before using this function. * @return Line (including newline) or false on eof @@ -1050,8 +1066,8 @@ static function time() /** * Output formatted and localized date - * @param format optional format (default is 2007-01-02 03:04:05). - * Other formats are "date", "datetime", "time". + * @param format optional format to be passed to date(), default "Y-m-d H:i:s" + * Other formats are "date", "datetime" and "time" in local language * Formats can be qualified with language, e.g. "date:en" * Special formats without language support are "icsdate", "icsdatetime". * @param stamp optional unix timestamp (default is now). @@ -1266,7 +1282,7 @@ static function json_encode($data, $p = []) */ static function json_decode($json, $p = []) { - return ($data = json_decode($json, $p['assoc'])) === null && trim($json) != 'null' ? it::error((array)$p['it_error'] + ['title' => "invalid json: " . json_last_error(), 'body' => $json]) : $data; + return ($data = json_decode($json, $p['assoc'])) === null && trim($json) != 'null' ? it::error((array)$p['it_error'] + ['title' => "invalid json: " . json_last_error_msg(), 'body' => $json]) : $data; } /** @@ -1302,6 +1318,7 @@ static function safe_filename($filename) { if (it::match("\.\./", $filename)) it::fatal(['title' => "../ contained in '$filename', aborted"]); + $filename = it::replace(['^/dev/fd/(\d+)$' => 'php://fd/$1'], $filename); return $filename; } diff --git a/it_dbi.class b/it_dbi.class index bd3e2e8..df21396 100644 --- a/it_dbi.class +++ b/it_dbi.class @@ -35,7 +35,7 @@ class it_dbi implements Iterator 'server_update' => null, # server to use for write and subsequent reads (only affects current object!) 'user' => "itools", 'pw' => "", - 'safety' => 1, # 0= never die, 1=die if query invalid, 2=die also if no results + 'safety' => 1, # -1=internal without it_error, 0= never die, 1=die if query invalid, 2=die also if no results #'keyfield' => 'ID', # Don't set to null here, filled later by _get_field_info() #'charset' => # client charset (requires MySQL 5.0.7 or later) 'classprefix' => "", @@ -501,9 +501,9 @@ function _where($params) /** * Internal: Output class name::error message and terminate execution. */ -function _fatal($text, $body = null) +function _fatal($text, $body = null, $fatal = true) { - it::fatal(['title' => $this->_error($text) . ", DB: " . $this->_p['db'] . ", Server: " . $this->_p['server'], 'body' => $body]); + it::error(['fatal' => $fatal, 'title' => $this->_error($text) . ", DB: " . $this->_p['db'] . ", Server: " . $this->_p['server'], 'body' => $body]); /* NOT REACHED */ } @@ -580,9 +580,10 @@ function query($query, $p = array()) if (!($result = $this->_query($query, $p))) { - if ($result === null || !$p['safety']) + if ($result === null || $p['safety'] < 0) return false; - $this->_fatal("query() failed", $query); + $this->_fatal("query() failed", $query, $p['safety']); + return false; } else if (it::match('^(CREATE|ALTER|DROP) ', $query, array('utf8' => false))) { @@ -828,7 +829,7 @@ function store($tags = array()) * Update current record or a number of records given by where condition * @param $tags key => value pairs (these have priority over changes in member vars) * @param $where condition to select records to be modified (if not current record) - * @return number of modified records (or false on error). WARNING: read LIMIT docs before using it + * @return number of modified records (or false on error). can be 0 if data already there. also read LIMIT docs * Does not destroy internal state of last select() call */ function update($tags = array(), $where = null) @@ -1121,7 +1122,7 @@ function _connect_db($p) function _get_field_defs() { - for ($res = $this->query('SHOW COLUMNS FROM ' . $this->_p['table']); $res && ($field = $this->_fetch_assoc($res)); ) + for ($res = $this->query('SHOW COLUMNS FROM ' . $this->_p['table'], ['safety' => -1]); $res && ($field = $this->_fetch_assoc($res)); ) $result[$field['Field']] = it::filter_keys($field, ['Field', 'Type', 'Key', 'Extra']); return (array)$result; } diff --git a/it_debug.class b/it_debug.class index 6cad9b5..8284093 100644 --- a/it_debug.class +++ b/it_debug.class @@ -120,6 +120,7 @@ static function setup() * @param $p['color'] Allow color [true] * @param $p['html'] Allow html [default based on REMOTE_ADDR] * @param $p['short'] Omit variable names + * @param $p['print_r'] Use print_r instead of var_export (no escaping for strings) * @return String representation of dump */ static function dump($args, $p = []) @@ -170,7 +171,7 @@ static function dump($args, $p = []) foreach ($args as $arg) { $var = array_shift($argnames); - $item = gettype($arg) == 'resource' || is_array($arg) && isset($arg['GLOBALS']) || is_object($arg) && is_a($arg, "SimpleXMLElement") ? trim(print_r($arg, true)) : trim(var_export($arg, true)); + $item = gettype($arg) == 'resource' || is_array($arg) && isset($arg['GLOBALS']) || is_object($arg) && is_a($arg, "SimpleXMLElement") || $p['print_r'] ? trim(print_r($arg, true)) : trim(var_export($arg, true)); # Replace PHP 5+ var_export object representation by old readable style while (preg_match("#(.*\b)(\w+)::__set_state\(array\(([^()]+)\)\)(.*)#s", $item, $regs) && ++$iterations < 100) diff --git a/it_mail.class b/it_mail.class index 11a4252..3d57bc3 100644 --- a/it_mail.class +++ b/it_mail.class @@ -575,22 +575,17 @@ static function check_email($email, $checkmailbox = false) $finished = false; $connected = 0; - if (function_exists('stream_set_timeout')) - $timeout = 'stream_set_timeout'; - else - $timeout = 'socket_set_timeout'; - foreach ($mx as $mxhost => $dummy_weight) { if ($fp = @fsockopen($mxhost, $port, $dummy_errno, $dummy_errstr, 5)) { $connected++; - $timeout($fp, 45); + stream_set_timeout($fp, 45); $answer = ''; if (it_mail::send_smtp_cmd($fp, '', $answer) && it_mail::send_smtp_cmd($fp, "HELO $fromdomain", $answer) && it_mail::send_smtp_cmd($fp, "MAIL FROM: <$from>", $answer)) { - $timeout($fp, 2); + stream_set_timeout($fp, 2); $timeoutok = ($domain != 'bluewin.ch'); if (it_mail::send_smtp_cmd($fp, "RCPT TO: <$email>", $answer, $timeoutok, 500)) # 450 is often used for Greylisting @@ -602,7 +597,7 @@ static function check_email($email, $checkmailbox = false) $finished = true; } - $timeout($fp, 0, 1); + stream_set_timeout($fp, 0, 1); it_mail::send_smtp_cmd($fp, 'RSET', $answer); diff --git a/it_url.class b/it_url.class index 8af61ae..50d7152 100644 --- a/it_url.class +++ b/it_url.class @@ -100,6 +100,8 @@ static function _postprocess($data, $p) * @param $p['filemtime'] Add HTTP header to only fetch when newer than this, otherwise return true instead of data * @param $p['accept_encoding'] Contents of the "Accept-Encoding: " header. Enables decoding of the response. Set to null to disable, "" (default) for all supported encodings. * @param $p['protocols'] Array of protocols to accept, defaults to ['http', 'https'], @see curl_opts for other values + * @param $p['user'] Username for basic HTTP authentication + * @param $p['pass'] Password for basic HTTP authentication * * Problem handling * @param $p['retries'] Number of retries if download fails, default 1 @@ -262,6 +264,7 @@ static function curl_opts($p=array()) $add += [CURLOPT_ENCODING => $p['accept_encoding']]; # NOTE: the curl library renamed the option to CURLOPT_ACCEPT_ENCODING, in php both are possible, CURLOPT_ENCODING is documented return $add + [ + CURLOPT_FRESH_CONNECT => $p['fresh_con'] ? true : false, CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT_MS => $p['totaltimeout'] * 1000, # use _MS to support fractions of seconds @@ -287,6 +290,7 @@ static function curl_opts($p=array()) * @param $p['data'] POST data array with key-value pairs * @param $p['files'] [fieldname => filename] of files to upload * @param $p['method'] different HTTP method + * @param $p['fresh_con'] force a fresh connection instead of a cached one * @param $p['verbose'] generate and capture curl verbose output in $this->verbose and alert mails */ @@ -314,8 +318,6 @@ function request($p=array()) curl_setopt($curl, CURLOPT_URL, $url->url); } - - # FIXME 2025-01 NG just use CURLOPT_MAXFILESIZE if we have curl 8.4 $content = ""; if ($p['maxlength'] && !$p['writefunction']) { @@ -339,7 +341,7 @@ function request($p=array()) $body = $origbody = $p['maxlength'] && $got ? $content : $got; $this->curlinfo = curl_getinfo($curl); - EDC('curlinfo', $this->curlinfo); + EDC('curlinfo', $this->curlinfo, substr($got, 0, 2048)); if ($body !== false || curl_errno($curl) == 23) { @@ -435,7 +437,6 @@ static function get_multi($p=null) }; $closehandle = function ($key) use (&$keys, &$handles, $mh) { curl_multi_remove_handle($mh, $handles[$key]); - curl_close($handles[$key]); unset($keys[(int)$handles[$key]]); unset($handles[$key]); }; @@ -513,26 +514,29 @@ static function get_multi($p=null) unset($urls[$key]); $closehandle($key); } - - if (!$abort && count($handles) < $parallel && $iterator->valid()) - { - $addhandle($iterator->key(), $iterator->current()); - $iterator->next(); - } } } } while ($mrc == CURLM_CALL_MULTI_PERFORM); - foreach ((array)$sleepuntils as $key => $time) - { - if (microtime(true) >= $time && count($handles) < $parallel) + if (!$abort) { + foreach ((array)$sleepuntils as $key => $time) { - $addhandle($key, $urls[$key]); - unset($sleepuntils[$key]); + if (microtime(true) >= $time && count($handles) < $parallel) + { + $addhandle($key, $urls[$key]); + unset($sleepuntils[$key]); + } } - $active = 1; + + while (count($handles) < $parallel && $iterator->valid()) + { + $addhandle($iterator->key(), $iterator->current()); + $iterator->next(); + } + + if ($sleepuntils && !count($handles)) + usleep(100000); } - usleep($sleepuntils ? 100000 : 0); $timeout = 0.1; # Longer delay to avoid busy loop but shorter than default of 1s in case we stil hit cURL 7.25.0 problem } @@ -574,7 +578,7 @@ static function get_cache_filename($p) $p['cachedir'] = it_url::get_cache_dir($p); unset($p['headers']['Authorization']); # prevent ever changing filenames due to changing Bearer tokens - $filename = $p['cachefilename'] ?: md5(T_lang() . T_defaultlang() . $p['url'] . ($p['headers'] ? serialize($p['headers']) : "") . ($p['data'] ? serialize($p['data']) : "") . $_SERVER['HTTP_X_SERVICE_PATH']); + $filename = $p['cachefilename'] ?: md5(T_lang() . T_defaultlang() . $p['url'] . ($p['headers'] ? serialize($p['headers']) : "") . ($p['data'] ? serialize($p['data']) : "") . $_SERVER['HTTP_X_SERVICE_PATH'] . $p['assoc']); return $p['cachedir'] . "/" . substr($filename, 0, 2) . "/$filename"; } @@ -710,8 +714,8 @@ static function get_cache($p = array()) $isnight = date('H') >= 1 && date('H')*3600 + date('i')*60 < $p['cleanbefore']; if (time() - @filemtime($p['cachedir'] . "/cleaned") > ($isnight ? 80000 : 2*80000)) { - it::file_put($p['cachedir'] . "/cleaned", ""); # touch could have permission problems - $maxagemin = intval($p['maxage']/60); + $maxagemin = round($p['maxage']/60, 2); + it::file_put($p['cachedir'] . "/cleaned", "$maxagemin\n"); exec("nohup bash -c 'cd {$p['cachedir']} && for i in [0-9a-f][0-9a-f]; do sleep 20; ionice -c 3 find \$i -mmin +$maxagemin -type f -delete; done' </dev/null >/dev/null 2>&1 &"); } diff --git a/it_xml.class b/it_xml.class index f74f743..d58f020 100644 --- a/it_xml.class +++ b/it_xml.class @@ -103,7 +103,6 @@ function from_xml($xmldata, $p) } unset($this->_arrayforce, $this->_p['safety'], $this->_p['it_error'], $this->_p['factory'], $this->_stack); - xml_parser_free($parser); return empty($this->error); } diff --git a/test/it_dbi.t b/test/it_dbi.t index b453da2..083047f 100755 --- a/test/it_dbi.t +++ b/test/it_dbi.t @@ -38,7 +38,7 @@ $opts['subclass']::createclass(['table' => "it_dbi_test", 'forcecreate' => true] $record = new it_dbi_test; $GLOBALS['it_defaultconfig']['fatal_throws_exception'] = true; -is($record->query("SYNTAX ERROR", ['safety' => 0]), false, "Suppress failures with safety 0"); +is(@$record->query("SYNTAX ERROR", ['safety' => 0]), false, "Suppress failures with safety 0"); try { is(@$record->select("SYNTAX ERROR"), "Exception", "Syntax triggers exception for fatal_throws_exception mode"); } catch (Exception $e) { diff --git a/test/it_html.t b/test/it_html.t index 20ab65f..13e2bc5 100755 --- a/test/it_html.t +++ b/test/it_html.t @@ -139,7 +139,7 @@ is( # XML generation unset($GLOBALS['it_html']); -new it_html(['htmltype' => "xml", 'name' => 'it_html', 'tags' => "xmltest", 'error_on_redefine' => false]); +new it_html(['htmltype' => "xml", 'name' => 'it_html', 'moretags' => "xmltest", 'error_on_redefine' => false]); is( xmltest(), diff --git a/test/it_url.testserver.php b/test/it_url.testserver.php index ca5300c..e59a81a 100644 --- a/test/it_url.testserver.php +++ b/test/it_url.testserver.php @@ -66,8 +66,12 @@ switch ($_SERVER['PHP_SELF']) break; case "/repeat": + if ($_REQUEST['compressed']) + ob_start('ob_gzhandler'); for ($i = 0; $i < $_REQUEST['num']; $i++) echo $_REQUEST['string']; + if ($_REQUEST['compressed']) + ob_end_flush(); break; case "/empty": diff --git a/test/it_url_slow.t b/test/it_url_slow.t index 00bbc2f..a5fd348 100755 --- a/test/it_url_slow.t +++ b/test/it_url_slow.t @@ -69,6 +69,13 @@ if (!$res || !$res2) handle_server( ok( + !it_url::get(['url' => "http://$host/repeat?string=abcdefghijklmnop&num=10&compressed", 'maxlength' => 100, 'retries' => 0, 'it_error' => false]), + 'it_url::get() fails for response larger than maxlength even if compressed response is smaller' + ) +); + +handle_server( + ok( it_url::get(U("http://$host/repeat", ['string' => "abc", 'num' => 1024 * 1024])) == str_repeat("abc", 1024 * 1024), 'it_url::get() handles large response' ) |