diff options
| -rw-r--r-- | it.class | 27 | ||||
| -rw-r--r-- | it_cache.class | 9 | ||||
| -rw-r--r-- | it_dbi.class | 20 | ||||
| -rw-r--r-- | it_debug.class | 3 | ||||
| -rw-r--r-- | it_html.class | 25 | ||||
| -rw-r--r-- | it_mail.class | 11 | ||||
| -rw-r--r-- | it_url.class | 66 | ||||
| -rw-r--r-- | it_xml.class | 1 | ||||
| -rwxr-xr-x | test/it_cache.t | 18 | ||||
| -rwxr-xr-x | test/it_dbi.t | 4 | ||||
| -rwxr-xr-x | test/it_dbi_postgres.t | 7 | ||||
| -rwxr-xr-x | test/it_html.t | 17 | ||||
| -rw-r--r-- | test/it_url.testserver.php | 4 | ||||
| -rwxr-xr-x | test/it_url_slow.t | 7 |
14 files changed, 137 insertions, 82 deletions
@@ -270,7 +270,7 @@ static function error($p = array(), $extra = null) if (strlen($p['body']) > 500000 || it::match('[\x00-\x08\x0E-\x1F]', $p['body'], ['utf8' => false]) !== null) { @file_put_contents($datafn = "$home/tmp/error-" . substr(md5($p['body']), 0, 2), $p['body']); # NOPHPLINT - $p['body'] = " See " . getenv('HOSTNAME') . ":$datafn"; + $p['body'] = "see " . getenv('HOSTNAME') . ":$datafn"; } $service = it::match('[^.]*', $GLOBALS['ULTRASITE']); @@ -1021,6 +1021,22 @@ static function _stdin_next() } /** + * Get all lines from command line arguments or stdin. + * Transparently reads existing files in the command line arguments and passes on all other arguments. + * Note: You need to call getopt() before using this function. + */ +static function gets_all() +{ + while (it::_stdin_next()) { + if (($fd = $GLOBALS['it_stdin']['fd'])) + while (($line = fgets($fd)) !== false) + yield rtrim($line, "\n"); + else + yield $GLOBALS['it_stdin']['filename']; + } +} + +/** * Get one line from stdin (or files given on command line) a la Perl <>. * Note: You need to call getopt() before using this function. * @return Line (including newline) or false on eof @@ -1050,8 +1066,8 @@ static function time() /** * Output formatted and localized date - * @param format optional format (default is 2007-01-02 03:04:05). - * Other formats are "date", "datetime", "time". + * @param format optional format to be passed to date(), default "Y-m-d H:i:s" + * Other formats are "date", "datetime" and "time" in local language * Formats can be qualified with language, e.g. "date:en" * Special formats without language support are "icsdate", "icsdatetime". * @param stamp optional unix timestamp (default is now). @@ -1266,7 +1282,7 @@ static function json_encode($data, $p = []) */ static function json_decode($json, $p = []) { - return ($data = json_decode($json, $p['assoc'])) === null && trim($json) != 'null' ? it::error((array)$p['it_error'] + ['title' => "invalid json: " . json_last_error(), 'body' => $json]) : $data; + return ($data = json_decode($json, $p['assoc'])) === null && trim($json) != 'null' ? it::error((array)$p['it_error'] + ['title' => "invalid json: " . json_last_error_msg(), 'body' => $json]) : $data; } /** @@ -1302,6 +1318,7 @@ static function safe_filename($filename) { if (it::match("\.\./", $filename)) it::fatal(['title' => "../ contained in '$filename', aborted"]); + $filename = it::replace(['^/dev/fd/(\d+)$' => 'php://fd/$1'], $filename); return $filename; } @@ -1349,7 +1366,7 @@ static function params2utf8() $_COOKIE = it::any2utf8($_COOKIE); $_FILES = it::any2utf8($_FILES); - foreach (['PHP_SELF', 'SCRIPT_NAME', 'SCRIPT_URL', 'SCRIPT_URI', 'HTTP_USER_AGENT'] as $var) + foreach (['PHP_SELF', 'SCRIPT_NAME', 'SCRIPT_URL', 'SCRIPT_URI', 'HTTP_USER_AGENT', 'HTTP_ACCEPT'] as $var) $_SERVER[$var] = it::any2utf8($_SERVER[$var]); $urlfix = function($m) { return urlencode(it::any2utf8(urldecode($m[0]))); }; # NOPHPLINT diff --git a/it_cache.class b/it_cache.class index 081db65..cec55dd 100644 --- a/it_cache.class +++ b/it_cache.class @@ -23,6 +23,7 @@ class it_cache { static $_fetch_func; static $_store_func; + static $_local = []; static function _defaults($p) { @@ -45,10 +46,10 @@ static function get($key, $p = array()) { $p = it_cache::_defaults($p); - if (isset($GLOBALS['it_cache_local'][$key])) + if (isset(it_cache::$_local[$key])) { # Use local copy - $result = $GLOBALS['it_cache_local'][$key]; + $result = it_cache::$_local[$key]; $success = true; } else if ($p['distributed'] && ($memcache = it_cache::_get_memcache($p))) @@ -85,14 +86,14 @@ static function put($key, $value, $p = array()) if ($memsuccess === false && $p['safety'] == 1) it::error(array_filter([ 'title' => ($p['distributed'] ? "memcache (" . ($memcache ? ($memcache->getResultMessage() . " on " . $memcache->getServerByKey($key)['host']) : "n/a") . ")" : self::$_store_func) . " in it_cache::put failed", - 'body' => "key='$key'", + 'body' => "key='$key', value size=" . strlen(json_encode($value)), 'id' => $p['distributed'] ? "it_cache_fail_" . $memcache->getServerByKey($key)['host'] : "it_cache_fail", 'timewindow' => "1200-1300", 'blockmailid' => $memcache ? "memcache_on_" . $memcache->getServerByKey($key)['host'] : null, 'blockmail' => $memcache? 12*3600 : null, ])); - $GLOBALS['it_cache_local'][$key] = $value; # Also store local copy + it_cache::$_local[$key] = $value; # Also store local copy return $value; } diff --git a/it_dbi.class b/it_dbi.class index bd3e2e8..a3d2c31 100644 --- a/it_dbi.class +++ b/it_dbi.class @@ -35,7 +35,7 @@ class it_dbi implements Iterator 'server_update' => null, # server to use for write and subsequent reads (only affects current object!) 'user' => "itools", 'pw' => "", - 'safety' => 1, # 0= never die, 1=die if query invalid, 2=die also if no results + 'safety' => 1, # -1=internal without it_error, 0= never die, 1=die if query invalid, 2=die also if no results #'keyfield' => 'ID', # Don't set to null here, filled later by _get_field_info() #'charset' => # client charset (requires MySQL 5.0.7 or later) 'classprefix' => "", @@ -287,7 +287,7 @@ function _expressions($tags, $force = false) if ($alldyns) { - if ($force == "insert") # INSERT/REPLACE + if (strval($force) == "insert") # INSERT/REPLACE $result['dyncols'] = $this->_json_object($alldyns); else if ($newdyns || $deldyns) { @@ -501,9 +501,9 @@ function _where($params) /** * Internal: Output class name::error message and terminate execution. */ -function _fatal($text, $body = null) +function _fatal($text, $body = null, $fatal = true) { - it::fatal(['title' => $this->_error($text) . ", DB: " . $this->_p['db'] . ", Server: " . $this->_p['server'], 'body' => $body]); + it::error(['fatal' => $fatal, 'title' => $this->_error($text) . ", DB: " . $this->_p['db'] . ", Server: " . $this->_p['server'], 'body' => $body]); /* NOT REACHED */ } @@ -580,9 +580,10 @@ function query($query, $p = array()) if (!($result = $this->_query($query, $p))) { - if ($result === null || !$p['safety']) + if ($result === null || $p['safety'] < 0) return false; - $this->_fatal("query() failed", $query); + $this->_fatal("query() failed", $query, $p['safety']); + return false; } else if (it::match('^(CREATE|ALTER|DROP) ', $query, array('utf8' => false))) { @@ -768,7 +769,8 @@ function iterate() /** * Insert a record into table. Values are taken assoc array $tags. Keys in $tags * should contain row names unless "dyncols" exists in schema (they are stored as - * dynamic columns then). Keys with - prefix suppress quoting of values. + * dynamic columns then). Keys with - prefix suppress quoting of values. If primary + * key field is (var)char and no value given, the value is generated randomly. * After inserting, all values are valid (record is read back). * Does not destroy internal state of last select() call * @param $tags key => value pairs to set @@ -828,7 +830,7 @@ function store($tags = array()) * Update current record or a number of records given by where condition * @param $tags key => value pairs (these have priority over changes in member vars) * @param $where condition to select records to be modified (if not current record) - * @return number of modified records (or false on error). WARNING: read LIMIT docs before using it + * @return number of modified records (or false on error). can be 0 if data already there. also read LIMIT docs * Does not destroy internal state of last select() call */ function update($tags = array(), $where = null) @@ -1121,7 +1123,7 @@ function _connect_db($p) function _get_field_defs() { - for ($res = $this->query('SHOW COLUMNS FROM ' . $this->_p['table']); $res && ($field = $this->_fetch_assoc($res)); ) + for ($res = $this->query('SHOW COLUMNS FROM ' . $this->_p['table'], ['safety' => -1]); $res && ($field = $this->_fetch_assoc($res)); ) $result[$field['Field']] = it::filter_keys($field, ['Field', 'Type', 'Key', 'Extra']); return (array)$result; } diff --git a/it_debug.class b/it_debug.class index 6cad9b5..8284093 100644 --- a/it_debug.class +++ b/it_debug.class @@ -120,6 +120,7 @@ static function setup() * @param $p['color'] Allow color [true] * @param $p['html'] Allow html [default based on REMOTE_ADDR] * @param $p['short'] Omit variable names + * @param $p['print_r'] Use print_r instead of var_export (no escaping for strings) * @return String representation of dump */ static function dump($args, $p = []) @@ -170,7 +171,7 @@ static function dump($args, $p = []) foreach ($args as $arg) { $var = array_shift($argnames); - $item = gettype($arg) == 'resource' || is_array($arg) && isset($arg['GLOBALS']) || is_object($arg) && is_a($arg, "SimpleXMLElement") ? trim(print_r($arg, true)) : trim(var_export($arg, true)); + $item = gettype($arg) == 'resource' || is_array($arg) && isset($arg['GLOBALS']) || is_object($arg) && is_a($arg, "SimpleXMLElement") || $p['print_r'] ? trim(print_r($arg, true)) : trim(var_export($arg, true)); # Replace PHP 5+ var_export object representation by old readable style while (preg_match("#(.*\b)(\w+)::__set_state\(array\(([^()]+)\)\)(.*)#s", $item, $regs) && ++$iterations < 100) diff --git a/it_html.class b/it_html.class index e3053b6..bab85da 100644 --- a/it_html.class +++ b/it_html.class @@ -393,32 +393,33 @@ static function sanitize($html) if ($charset == "utf-8") $html = it::any2utf8($html); $html = it::replace(array('[\0\s]+' => " "), $html); # \s also matches \r and \n - $urlpattern = 'https?://[^">]+'; + $urlpattern = '(?:https?://|mailto:)[^">]+'; + $placeholder = bin2hex(random_bytes(16)); if ($tag = it::match("(.*?)<(div|p|ol|ul|li|i|b|strong|h[1-6])\b[^>]*>(.*?)</\\2>(.*)", $html)) { # Simple tags with content, no attributes kept list($head, $tagname, $content, $tail) = $tag; $tagname = strtolower($tagname); - $result .= it_html::sanitize($head) . "<$tagname>" . it_html::sanitize($content) . "</$tagname>" . it_html::sanitize($tail); + $result .= it::replace([$placeholder => "<$tagname>" . it_html::sanitize($content) . "</$tagname>"], it_html::sanitize("$head$placeholder$tail")); } else if ($tag = it::match('(.*)<a\b[^>]+?\bhref\s*=\s*"(' . $urlpattern . ')"[^>]*?>(.*?)</a>(.*)', $html)) { # Link tags, keeps only href attribute list($head, $href, $content, $tail) = $tag; - $result .= it_html::sanitize($head) . '<a href="' . it_html::Q(it_html::U(html_entity_decode($href, ENT_COMPAT, $charset))) . '">' . it_html::sanitize($content) . "</a>" . it_html::sanitize($tail); + $result .= it::replace([$placeholder => '<a href="' . it_html::Q(it_html::U(html_entity_decode($href, ENT_COMPAT, $charset))) . '">' . it_html::sanitize($content) . "</a>"], it_html::sanitize("$head$placeholder$tail")); } else if ($tag = it::match('(.*)<img\b[^>]+?\bsrc\s*=\s*"(' . $urlpattern . ')"[^>]*?>(.*)', $html)) { # Image tags, keeps only src attribute list($head, $src, $tail) = $tag; - $result .= it_html::sanitize($head) . '<img src="' . it_html::Q(it_html::U(html_entity_decode($src, ENT_COMPAT, $charset))) . '" alt="" />' . it_html::sanitize($tail); + $result .= it::replace([$placeholder => '<img src="' . it_html::Q(it_html::U(html_entity_decode($src, ENT_COMPAT, $charset))) . '" alt="" />'], it_html::sanitize("$head$placeholder$tail")); } else if ($tag = it::match("(.*)<(br|/tr)\b[^>]*>(.*)", $html)) { # brs and table rows are converted so simple line breaks list($head, $tagname, $tail) = $tag; - $result .= it_html::sanitize($head) . "<br />" . it_html::sanitize($tail); + $result .= it::replace([$placeholder => "<br />"], it_html::sanitize("$head$placeholder$tail")); } else $result = it::replace(array('&(#\d+;)' => '&$1'), it_html::Q(html_entity_decode(strip_tags($html), ENT_COMPAT, $charset))); @@ -545,18 +546,16 @@ static function U(...$args) */ function js($args) { - $args = it::map(fn($v) => it::replace(['<!--' => '\\x3C!--', '<script' => '\\x3Cscript', '</script' => '\\x3C/script'], $v), $args); + list($base, $params) = it_parse_args($args); + $base= it::replace(['<!--' => '\\x3C!--', '<script' => '\\x3Cscript', '</script' => '\\x3C/script'], $base); - if (($this->p['htmltype'][0] == 'x') && $args[0] && ((array)$args[0] === array_values((array)$args[0]))) - { - array_unshift($args, "<!--//--><![CDATA[//><!--\n"); - $args[] = "\n//--><!]]>"; - } + if (($this->p['htmltype'][0] == 'x') && strlen($base)) + $base = "<!--//--><![CDATA[//><!--\n$base\n//--><!]]>"; if ($this->p['htmltype'] != "html5") - array_unshift($args, array('type' => 'text/javascript')); + $params['type'] = 'text/javascript'; - return $this->_tag('script', $args); + return $this->_tag('script', [$base, $params]); } diff --git a/it_mail.class b/it_mail.class index 11a4252..3d57bc3 100644 --- a/it_mail.class +++ b/it_mail.class @@ -575,22 +575,17 @@ static function check_email($email, $checkmailbox = false) $finished = false; $connected = 0; - if (function_exists('stream_set_timeout')) - $timeout = 'stream_set_timeout'; - else - $timeout = 'socket_set_timeout'; - foreach ($mx as $mxhost => $dummy_weight) { if ($fp = @fsockopen($mxhost, $port, $dummy_errno, $dummy_errstr, 5)) { $connected++; - $timeout($fp, 45); + stream_set_timeout($fp, 45); $answer = ''; if (it_mail::send_smtp_cmd($fp, '', $answer) && it_mail::send_smtp_cmd($fp, "HELO $fromdomain", $answer) && it_mail::send_smtp_cmd($fp, "MAIL FROM: <$from>", $answer)) { - $timeout($fp, 2); + stream_set_timeout($fp, 2); $timeoutok = ($domain != 'bluewin.ch'); if (it_mail::send_smtp_cmd($fp, "RCPT TO: <$email>", $answer, $timeoutok, 500)) # 450 is often used for Greylisting @@ -602,7 +597,7 @@ static function check_email($email, $checkmailbox = false) $finished = true; } - $timeout($fp, 0, 1); + stream_set_timeout($fp, 0, 1); it_mail::send_smtp_cmd($fp, 'RSET', $answer); diff --git a/it_url.class b/it_url.class index 8af61ae..48dbb6f 100644 --- a/it_url.class +++ b/it_url.class @@ -82,7 +82,10 @@ static function is_reachable($p = []) static function _postprocess($data, $p) { if ($p['postprocess']) - $data = ($t = $p['postprocess']($data, ['it_error' => $p['retries'] > 0 ? false : (array)$p['it_error'] + ['title' => "invalid content from " . $p['url']]])) && $p['checkonly'] ? $data : $t; + { + $processed = $p['postprocess']($data, ['it_error' => $p['retries'] > 0 ? false : (array)$p['it_error'] + ['title' => "invalid content from " . $p['url']]]); + $data = $processed !== null && $processed !== false && $p['checkonly'] ? $data : $processed; + } return $data; } @@ -100,6 +103,8 @@ static function _postprocess($data, $p) * @param $p['filemtime'] Add HTTP header to only fetch when newer than this, otherwise return true instead of data * @param $p['accept_encoding'] Contents of the "Accept-Encoding: " header. Enables decoding of the response. Set to null to disable, "" (default) for all supported encodings. * @param $p['protocols'] Array of protocols to accept, defaults to ['http', 'https'], @see curl_opts for other values + * @param $p['user'] Username for basic HTTP authentication + * @param $p['pass'] Password for basic HTTP authentication * * Problem handling * @param $p['retries'] Number of retries if download fails, default 1 @@ -114,7 +119,7 @@ static function _postprocess($data, $p) * Result processing * @param $p['assoc'] Return [ 'data' => string, 'status' => int, 'cookies' => array, 'headers' => array, 'errstr' => string ] instead of just data * @param $p['writefunction'] function to be called whenever data is received (for server-sent-events etc.) - * @param $p['postprocess'] function called with content and $p which has it_error. returns content or null (which triggers retry) + * @param $p['postprocess'] function called with content and $p which has it_error params. returns content or null/false (which triggers retry) * @param $p['followlocation']Follow redirects [true] * * @return Content of resulting page (considering redirects, excluding headers or false on error) or array if 'assoc' => true @@ -165,7 +170,7 @@ function _get($p = []) static function retry_warranted($result, $status) { - return $result ? it::match(self::$forceretry, $status) : !it::match('^(204|4..)$', $status); + return $result || $result === [] ? it::match(self::$forceretry, $status) : !it::match('^(204|4..)$', $status); } function parse_http_header($header) @@ -184,13 +189,13 @@ function parse_http_header($header) static function _default_headers($url, $p) { - $search_subrequest = it::match('search\.ch/', $p['url']); + $search_subrequest = it::match('\bsearch\.ch/', $p['url']); if ((!it::is_devel() || EDC('subreqcheck')) && $p['url'] && !$p['headers']['Accept-Language'] && T_lang() != T_defaultlang() && $search_subrequest && !it::match('/login\b|banner\.html|machines\.txt|mbtiles\.php|/fonts/|/itjs/|/images/|\.(de|fr|en|it)(\.js|\.html|\.txt|\.php|\.ics|\.pdf|\.json|\.csv|\.gif|\.jpg|\.png)', $p['url'])) it::error(['title' => "Subrequest without language override", 'body' => [ $p ]]); $headers = array_filter([ 'Host' => $url->realhostname . $url->explicitport, - 'User-Agent' => "Mozilla/5.0 (compatible; ITools; Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582)", + 'User-Agent' => "Mozilla/5.0 (compatible; ITools; Chrome/141.0.0.0 Safari/604.1)", 'Accept-Language' => $p['headers']['Accept-Language'] ?? ($search_subrequest ? T_defaultlang() : T_lang()), # can prevent loading of it_text 'Referer' => it::match('([-\w]+\.\w+)$', $url->hostname) == it::match('([-\w]+\.\w+)$', $_SERVER['HTTP_HOST']) ? it::replace(['%[0-9a-f]?$' => ''], substr(static::absolute(U($_GET)), 0, 8000)) : null, # Truncate overly long referers leading to failed subrequest but make sure it is still propery urlencoded 'X-Ultra-Https' => $_SERVER['HTTPS'], @@ -262,6 +267,7 @@ static function curl_opts($p=array()) $add += [CURLOPT_ENCODING => $p['accept_encoding']]; # NOTE: the curl library renamed the option to CURLOPT_ACCEPT_ENCODING, in php both are possible, CURLOPT_ENCODING is documented return $add + [ + CURLOPT_FRESH_CONNECT => $p['fresh_con'] ? true : false, CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT_MS => $p['totaltimeout'] * 1000, # use _MS to support fractions of seconds @@ -287,6 +293,7 @@ static function curl_opts($p=array()) * @param $p['data'] POST data array with key-value pairs * @param $p['files'] [fieldname => filename] of files to upload * @param $p['method'] different HTTP method + * @param $p['fresh_con'] force a fresh connection instead of a cached one * @param $p['verbose'] generate and capture curl verbose output in $this->verbose and alert mails */ @@ -314,8 +321,6 @@ function request($p=array()) curl_setopt($curl, CURLOPT_URL, $url->url); } - - # FIXME 2025-01 NG just use CURLOPT_MAXFILESIZE if we have curl 8.4 $content = ""; if ($p['maxlength'] && !$p['writefunction']) { @@ -339,7 +344,7 @@ function request($p=array()) $body = $origbody = $p['maxlength'] && $got ? $content : $got; $this->curlinfo = curl_getinfo($curl); - EDC('curlinfo', $this->curlinfo); + EDC('curlinfo', $this->curlinfo, substr($got, 0, 2048)); if ($body !== false || curl_errno($curl) == 23) { @@ -435,7 +440,6 @@ static function get_multi($p=null) }; $closehandle = function ($key) use (&$keys, &$handles, $mh) { curl_multi_remove_handle($mh, $handles[$key]); - curl_close($handles[$key]); unset($keys[(int)$handles[$key]]); unset($handles[$key]); }; @@ -513,26 +517,29 @@ static function get_multi($p=null) unset($urls[$key]); $closehandle($key); } - - if (!$abort && count($handles) < $parallel && $iterator->valid()) - { - $addhandle($iterator->key(), $iterator->current()); - $iterator->next(); - } } } } while ($mrc == CURLM_CALL_MULTI_PERFORM); - foreach ((array)$sleepuntils as $key => $time) - { - if (microtime(true) >= $time && count($handles) < $parallel) + if (!$abort) { + foreach ((array)$sleepuntils as $key => $time) { - $addhandle($key, $urls[$key]); - unset($sleepuntils[$key]); + if (microtime(true) >= $time && count($handles) < $parallel) + { + $addhandle($key, $urls[$key]); + unset($sleepuntils[$key]); + } + } + + while (count($handles) < $parallel && $iterator->valid()) + { + $addhandle($iterator->key(), $iterator->current()); + $iterator->next(); } - $active = 1; + + if ($sleepuntils && !count($handles)) + usleep(100000); } - usleep($sleepuntils ? 100000 : 0); $timeout = 0.1; # Longer delay to avoid busy loop but shorter than default of 1s in case we stil hit cURL 7.25.0 problem } @@ -574,7 +581,7 @@ static function get_cache_filename($p) $p['cachedir'] = it_url::get_cache_dir($p); unset($p['headers']['Authorization']); # prevent ever changing filenames due to changing Bearer tokens - $filename = $p['cachefilename'] ?: md5(T_lang() . T_defaultlang() . $p['url'] . ($p['headers'] ? serialize($p['headers']) : "") . ($p['data'] ? serialize($p['data']) : "") . $_SERVER['HTTP_X_SERVICE_PATH']); + $filename = $p['cachefilename'] ?: md5(T_lang() . T_defaultlang() . $p['url'] . ($p['headers'] ? serialize($p['headers']) : "") . ($p['data'] ? serialize($p['data']) : "") . $_SERVER['HTTP_X_SERVICE_PATH'] . $p['assoc']); return $p['cachedir'] . "/" . substr($filename, 0, 2) . "/$filename"; } @@ -633,14 +640,15 @@ static function get_cache($p = array()) { $success = true; $isnewfile = it_url::_atomicwrite($path, $p['assoc'] ? ($data['status'] === 304 ? true : it::json_encode($data)) : $data); # $data === true means not modified (no new data fetched) and instructs _atomicwrite to just touch the file - if ($p['returnheaders']) - it::file_put("$path.json", it::json_encode($url->headers)); } else if ($p['keepfailed']) - $success = $fileexists; + $success = $kept = $fileexists; else @unlink($path); # Expired and failed to get + if ($p['returnheaders'] && !$kept) + it::file_put("$path.json", it::json_encode($url->headers)); + it_url::_unlock($path, $lock); } else @@ -710,8 +718,8 @@ static function get_cache($p = array()) $isnight = date('H') >= 1 && date('H')*3600 + date('i')*60 < $p['cleanbefore']; if (time() - @filemtime($p['cachedir'] . "/cleaned") > ($isnight ? 80000 : 2*80000)) { - it::file_put($p['cachedir'] . "/cleaned", ""); # touch could have permission problems - $maxagemin = intval($p['maxage']/60); + $maxagemin = round($p['maxage']/60, 2); + it::file_put($p['cachedir'] . "/cleaned", "$maxagemin\n"); exec("nohup bash -c 'cd {$p['cachedir']} && for i in [0-9a-f][0-9a-f]; do sleep 20; ionice -c 3 find \$i -mmin +$maxagemin -type f -delete; done' </dev/null >/dev/null 2>&1 &"); } @@ -764,7 +772,7 @@ static function _expired($path, $maxage, $randomexpire = 0) else $result = false; } - else # File does not exists yet + else # File does not exist yet $result = true; return $result; diff --git a/it_xml.class b/it_xml.class index f74f743..d58f020 100644 --- a/it_xml.class +++ b/it_xml.class @@ -103,7 +103,6 @@ function from_xml($xmldata, $p) } unset($this->_arrayforce, $this->_p['safety'], $this->_p['it_error'], $this->_p['factory'], $this->_stack); - xml_parser_free($parser); return empty($this->error); } diff --git a/test/it_cache.t b/test/it_cache.t index e2e94bf..3e2fb0d 100755 --- a/test/it_cache.t +++ b/test/it_cache.t @@ -14,15 +14,15 @@ is(it_cache::get('it_cache_t'), [2], "cache put/get array"); # test non-distributed apc cache it_cache::put('it_cache_t', 42); -unset($GLOBALS['it_cache_local']); +it_cache::$_local = []; is(it_cache::get('it_cache_t'), 42, "local put/get number"); it_cache::put('it_cache_t', false); -unset($GLOBALS['it_cache_local']); +it_cache::$_local = []; is(it_cache::get('it_cache_t'), false, "local put/get false"); it_cache::put('it_cache_t', [2]); -unset($GLOBALS['it_cache_local']); +it_cache::$_local = []; is(it_cache::get('it_cache_t'), [2], "local put/get array"); is(it_cache::get('it_cache_t'.rand(1, 1000)), null, "local get unknown key"); @@ -32,18 +32,18 @@ is(it_cache::get('it_cache_t'.rand(1, 1000)), null, "local get unknown key"); $GLOBALS['debug_aslive'] = 1; it_cache::put('it_cache_d', 42, ['distributed' => 1]); -it_cache::put('it_cache_d', 0); -unset($GLOBALS['it_cache_local']); +it_cache::put('it_cache_d', 1); +it_cache::$_local = []; is(intval(it_cache::get('it_cache_d', ['distributed' => 1])), 42, "distributed put/get number"); it_cache::put('it_cache_d', false, ['distributed' => 1]); -it_cache::put('it_cache_d', 1); -unset($GLOBALS['it_cache_local']); +it_cache::put('it_cache_d', 2); +it_cache::$_local = []; is(boolval(it_cache::get('it_cache_d', ['distributed' => 1])), false, "distributed put/get false"); it_cache::put('it_cache_d', [2], ['distributed' => 1]); -it_cache::put('it_cache_d', 0); -unset($GLOBALS['it_cache_local']); +it_cache::put('it_cache_d', 3); +it_cache::$_local = []; is(it_cache::get('it_cache_d', ['distributed' => 1]), [2], "distributed put/get array"); is(it_cache::get('it_cache_d'.rand(1, 1000), ['distributed' => 1]), null, "distributed get unknown key"); diff --git a/test/it_dbi.t b/test/it_dbi.t index b453da2..1140c55 100755 --- a/test/it_dbi.t +++ b/test/it_dbi.t @@ -38,7 +38,7 @@ $opts['subclass']::createclass(['table' => "it_dbi_test", 'forcecreate' => true] $record = new it_dbi_test; $GLOBALS['it_defaultconfig']['fatal_throws_exception'] = true; -is($record->query("SYNTAX ERROR", ['safety' => 0]), false, "Suppress failures with safety 0"); +is(@$record->query("SYNTAX ERROR", ['safety' => 0]), false, "Suppress failures with safety 0"); try { is(@$record->select("SYNTAX ERROR"), "Exception", "Syntax triggers exception for fatal_throws_exception mode"); } catch (Exception $e) { @@ -439,6 +439,8 @@ $r->iterate(); is($r->_dyndata, [], '_dyndata for record with empty dyncols should be empty'); $r->update(['key3' => 'c']); is($r->key3, 'c', 'dynamic column for record with empty dyncols whould be correctly created'); +$r->update(['key3' => null], ['ID' => 3]); +is($r->key3, null, 'remove dynamic column completely if where-clause is given'); $r->clear(false); is($r->select(['-key1 IS NOT' => 'NULL']), 1, 'only one entry has a value for key1'); diff --git a/test/it_dbi_postgres.t b/test/it_dbi_postgres.t index a272215..5057d45 100755 --- a/test/it_dbi_postgres.t +++ b/test/it_dbi_postgres.t @@ -1,2 +1,7 @@ #!/bin/sh -systemctl check -q postgresql.service && `dirname $0`/it_dbi.t --subclass it_dbi_postgres --db map_search_ch
\ No newline at end of file + +if systemctl check -q postgresql.service || ! servertype live; then + `dirname $0`/it_dbi.t --subclass it_dbi_postgres --db map_search_ch +else + echo ok 1 - Skipping tests on live because postgresql is not running +fi diff --git a/test/it_html.t b/test/it_html.t index 15f444d..90af4b6 100755 --- a/test/it_html.t +++ b/test/it_html.t @@ -81,6 +81,8 @@ is( "leave legal utf8 intact" ); +is(js(['async' => true], 'foo'), "<script async>foo</script>\n", 'boolean attribute for js script tag'); + unset($GLOBALS['debug_utf8check']); is( div(['arg' => "value \xc2", "content"]), @@ -136,10 +138,11 @@ is( "xhtml doctype" ); +is(js('foo'), "<script type=\"text/javascript\"><!--//--><![CDATA[//><!--\nfoo\n//--><!]]></script>\n", "escape js script content with CDATA in xml mode"); # XML generation unset($GLOBALS['it_html']); -new it_html(['htmltype' => "xml", 'name' => 'it_html', 'tags' => "xmltest", 'error_on_redefine' => false]); +new it_html(['htmltype' => "xml", 'name' => 'it_html', 'moretags' => "xmltest", 'error_on_redefine' => false]); is( xmltest(), @@ -260,6 +263,18 @@ is( ); is( + it_html::sanitize('<a href="http://search.ch/"><strong>foo</strong></a>'), + '<a href="http://search.ch/"><strong>foo</strong></a>', + 'it_html::sanitize handle nesting of tags inside <a>' +); + +is( + it_html::sanitize('<a href="mailto:neuman@example.com">foo</a>'), + '<a href="mailto:neuman@example.com">foo</a>', + 'it_html::sanitize handle mailto links' +); + +is( it_html::sanitize("<a href='http://search.ch/'>foo</a>"), '<a href="http://search.ch/">foo</a>', 'TODO it_html::sanitize handle anchors with single quotes at attribute value' diff --git a/test/it_url.testserver.php b/test/it_url.testserver.php index ca5300c..e59a81a 100644 --- a/test/it_url.testserver.php +++ b/test/it_url.testserver.php @@ -66,8 +66,12 @@ switch ($_SERVER['PHP_SELF']) break; case "/repeat": + if ($_REQUEST['compressed']) + ob_start('ob_gzhandler'); for ($i = 0; $i < $_REQUEST['num']; $i++) echo $_REQUEST['string']; + if ($_REQUEST['compressed']) + ob_end_flush(); break; case "/empty": diff --git a/test/it_url_slow.t b/test/it_url_slow.t index 00bbc2f..a5fd348 100755 --- a/test/it_url_slow.t +++ b/test/it_url_slow.t @@ -69,6 +69,13 @@ if (!$res || !$res2) handle_server( ok( + !it_url::get(['url' => "http://$host/repeat?string=abcdefghijklmnop&num=10&compressed", 'maxlength' => 100, 'retries' => 0, 'it_error' => false]), + 'it_url::get() fails for response larger than maxlength even if compressed response is smaller' + ) +); + +handle_server( + ok( it_url::get(U("http://$host/repeat", ['string' => "abc", 'num' => 1024 * 1024])) == str_repeat("abc", 1024 * 1024), 'it_url::get() handles large response' ) |