Merge branch 'master' into ng/postgresql

11 files changed, 116 insertions, 134 deletions

diff --git a/it.class b/it.class
index 0d3cdec..798e774 100644
--- a/it.class
+++ b/it.class
@@ -155,17 +155,20 @@ static function timerlog($label = '')
  *                                                             |graceperiod|timewindow|------>
  *   id             x  x     x       m    x   x                x     x
  */
-static function error($p = array())
+static function error($p = array(), $extra = null)
 {
 	$p = $origp = (array)$p;
 	$p['title'] = $p[0] ?: $p['title']; # handle 'it_error' => "oops" that was cast to array on the way
 	$p['title'] = grapheme_substr($p['title'], 0, 2000);
 
+	if ($extra)
+		it::error('extraneous params passed to it::error');
+
 	foreach (array_slice(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 10), 1) as $level)
 		if ($level["class"] . $level["type"] . $level["function"] == "it::error")
 			return null; # prevent recursion
 
-	if (!error_reporting() || $p[0] === false || $p['title'] === false) # called with @ or suppressed
+	if (error_reporting() == @error_reporting() || $p[0] === false || $p['title'] === false) # called with @ or suppressed
 		return $p['fatal'] ? self::_exit($p) : null;
 
 	if ($_SERVER['REMOTE_ADDR'])
@@ -260,9 +263,9 @@ static function error($p = array())
 			($p['omitdebuginfo'] >= 2 ? "" : ($url && !$toscreen? "Title:  {$p['title']}\nUrl:    $url\n" : "") .
 			($trace ? ($sendmail ? "" : "  ") . "Trace:  $trace\n" : "")) .
 			(!$sendmail || $p['omitdebuginfo'] >= 2 ? "" : "Host:   " . getenv('HOSTNAME') . " at " . date("Y-m-d H:i:s") . (($t = time() - $_SERVER['REQUEST_TIME']) ? " (invoked {$t}s before)" : "") . "\n") . # no it::date() due to time- debug param
-			self::$error_context .
+			($sendmail ? self::$error_context : "") .
 			($p['id']     ? "Filter: timewindow=" . $p['graceperiod'] . "-" . ($p['graceperiod'] + $p['timewindow']) . "  (previous err: " . it::date('', $errstamp) . ")\n" : "") .
-			(!$origp['blockmail'] || $p['omitdebuginfo'] ? "" : "Block-resend: " . $origp['blockmail'] . " seconds\n") .
+			(!$origp['blockmail'] || $p['omitdebuginfo'] || $toscreen ? "" : "Block-resend: " . $origp['blockmail'] . " seconds\n") .
 			($p['body'] ? ($p['omitdebuginfo'] ? "" : "Body:\n") . trim($p['body'])."\n\n" : "");
 
 		if ($sendmail) # we're mailing: send maximum info
@@ -275,10 +278,13 @@ static function error($p = array())
 					if ($level >= $p['backtraceskip'] && $tracesize < 100000 && ($tracesize += strlen(print_r($stackframe, true))) < 100000) # save mem
 						$stackframes[] = $stackframe;
 
+				$rawpost = $_SERVER['REQUEST_METHOD'] == "POST" && !it::match('application/x-www-form-urlencoded|multipart/form-data', $_SERVER['CONTENT_TYPE']) ? it::file_get_contents("php://input") : "";
+
 				$body .= ($trace and $t = it::exec('grep -h {0} `ls /tmp/alertdata/alertlog-*|tail -3` /dev/null 2>/dev/null | grep ^2 | cut -d : -f 1-2 | sort | uniq -c | tail -10', $trace)) ? "Histogram: (last 10 affected minutes in 3 days)\n$t" : "";
 				$body .= "\n";
 				$body .= $_GET        ? "\$_GET: "    . var_export($_GET, true) . "\n\n" : "";
 				$body .= $_POST       ? "\$_POST: "   . var_export($_POST, true) . "\n\n" : "";
+				$body .= $rawpost     ? "\$rawpost: " . var_export($rawpost, true) . "\n\n" : "";
 				$body .= $_COOKIE     ? "\$_COOKIE: " . var_export($_COOKIE, true) . "\n\n" : "";
 				$body .= $_SERVER['REMOTE_ADDR'] ? "" : "Pstree:\n" . it::exec("pstree -als " . getmypid() . " | head -n -3") . "\n";
 				$body .= $_SERVER     ? "\$_SERVER: " . var_export($_SERVER, true) . "\n\n" : "";
@@ -547,7 +553,7 @@ static function any2utf8($value, $errprefix = "")
 		if (grapheme_strlen($value) === null)
 			list($value, $error) = array(utf8_encode($value), utf8_encode("incorrect utf8-encoding. input=$value"));
 		if (preg_match('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', $value))
-			list($value, $error) = array(it::any2utf8(preg_replace_callback('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', function($m) {return utf8_decode($m[0]);}, $value)), $errprefix ? "$errprefix: double utf8-encoding. input=$value" : "");
+			list($value, $error) = array(it::any2utf8(preg_replace_callback('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', function($m) {return utf8_decode($m[0]);}, $value)), $errprefix ? "double utf8-encoding. input=$value" : "");
 		if (preg_match('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', $value))
 			list($value, $error) = array(preg_replace('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', " ", $value), "forbidden utf-8 character. input=$value");
 		$value = preg_replace('/\xc2\xad/', '', $value);	# Kill invisible soft hyphens
@@ -797,7 +803,7 @@ static function imageconvert($p)
 	$ultratimeout = file_exists("/opt/ultra/bin/ultratimeout") ? "/opt/ultra/bin/ultratimeout 30 " : "";
 
 	if (in_array($type, explode(',', $p['types'])))	# Valid type?
-		$cmdoutput = it::exec('( ' . $ultratimeout . 'gm convert 2>&1 {-opts} {in} {type}:{out} || echo "SHELL ERROR $?" ) | grep -v " iCCP: "', $p);
+		$cmdoutput = it::exec('( ' . $ultratimeout . 'gm convert -limit threads 2 2>&1 {-opts} {in} {type}:{out} || echo "SHELL ERROR $?" ) | grep -Ev "( iCCP: | Invalid SOS parameters for sequential JPEG | profile matches .* but writing .* instead | inconsistent chromacities )"', $p);
 
 	if ($p['pngcrush'] && $p['type'] == "png")
 		it::exec('pngcrush.sh 2>/dev/null {out} {out}.tmp && mv {out}.tmp {out} || rm {out}.tmp', $p);
@@ -925,11 +931,10 @@ static function getopt($usage, $p = array())
 			$noopts = true;
 	}
 
-	if (!$optionalargs && $result['args'])
-		it::error("Optional arguments passed to script without optional args in usage"); # FIXME 2020-10 NG merge with normal usage errors below
-	if ($err || $eat || $result['h'] || $result['help'] || $mandatoryargs)
+	if ($err || $eat || $result['h'] || $result['help'] || $mandatoryargs || (!$optionalargs && $result['args']))
 	{
-		fputs(($result['h'] || $result['help'] ? STDOUT : STDERR), trim($usage) . "\n");
+		if (is_resource($out = $result['h'] || $result['help'] ? STDOUT : STDERR))
+			fputs($out, trim($usage) . "\n");
 		return $p['noexit'] ? false : exit(1);
 	}
 
@@ -1029,6 +1034,9 @@ static function date($format = "", $stamp = null)
 # Internal: Convert expression or funcname or function literal to callable
 static function createfunc($code)
 {
+	if ($code instanceof Closure && (new ReflectionFunction($code))->getNumberOfRequiredParameters() == 1)
+		$code = function($dummy, $v) use ($code) {return $code($v);};
+
 	if (is_string($code) && it::match('^[\w:]+$', $code) && is_callable($code))
 		$code .= '($v)';
 
diff --git a/it_dbi.class b/it_dbi.class
index 6383910..e51b23e 100644
--- a/it_dbi.class
+++ b/it_dbi.class
@@ -151,8 +151,8 @@ static function createclass($p)
 		$p = array('table' => $p);
 
 	# Make sure singleton exists
-	$dbi = $GLOBALS[static::$_global_key] ?: new static::$_global_key(['table' => null] +  $p);
-	$p += $dbi->_p;	# FIXME: (has to be checked for side effects!)
+	$dbi = $GLOBALS[static::$_global_key] ? $GLOBALS[static::$_global_key] : new static::$_global_key(['table' => null] +  $p);
+	$p += $dbi->_p;
 	$dbid = $dbi->_dbid = "{$p['user']}@{$p['server']}:{$p['db']}";
 
 	if (!isset($dbi->_tables[$dbid]))
@@ -488,26 +488,6 @@ function _fatal($text, $body = null)
 }
 
 /**
- * Post-process data after reading a record.
- * This is a stub-function that can be overloaded.
- */
-function _read_post_process()
-{
-}
-
-
-/**
- * Pre-process data before writing a record.
- * This is a stub-function that can be overloaded.
- * @param $tags Reference to update/create tags, can be modified as needed
- * @param $command SQL command ('INSERT', 'REPLACE' or 'UPDATE')
- */
-function _write_pre_process(&$tags, $command)	# NOPHPLINT FIXME 2020-12 CS Remove legacy hooks now replace by pure and static versions
-{
-}
-
-
-/**
  * Hook to postprocess data after reading a record.
  * This is a stub-function that can be overloaded.
  * @param $data Data of record read from db
@@ -592,7 +572,10 @@ function query($query, $p = array())
 	}
 
 	if ($writing && $this->_p['throttle_writes'])
+	{
+		it::log('debug', 'dbi-throttle', 1000000 * (gettimeofday(true) - $start) * $this->_p['throttle_writes']);
 		usleep(1000000 * (gettimeofday(true) - $start) * $this->_p['throttle_writes']);
+	}
 
 	$msec = round(1000 * (gettimeofday(true) - $start));
 	$slow = $msec >= 2000;
@@ -756,8 +739,6 @@ function iterate()
 			if ($this->_p['unbuffered'])
 				$this->_result->close();
 		}
-
-		$this->_read_post_process();
 	}
 	else
 		$this->_nofetch = false;
@@ -781,7 +762,6 @@ function insert($tags = array(), $command = "INSERT", $suffix = "")
 
 	/* Pre-processing, $tags is passed by reference and may be modified here */
 	$tags = static::_write_preprocess($tags);
-	$this->_write_pre_process($tags, $command);
 
 	if ($this->_p['randomid'] && !isset($tags[$this->_p['keyfield']]))
 		$tags[$this->_p['keyfield']] = bin2hex(random_bytes(16));
@@ -840,7 +820,6 @@ function update($tags = array(), $where = null)
 
 	/* Pre-processing, $tags is passed by reference and may be modified here */
 	$tags = static::_write_preprocess($tags);
-	$this->_write_pre_process($tags, 'UPDATE');
 
 	if ($set = $this->_set($tags, isset($where)))
 	{
diff --git a/it_debug.class b/it_debug.class
index 409b708..12fb254 100644
--- a/it_debug.class
+++ b/it_debug.class
@@ -129,7 +129,7 @@ static function dump($args)
 			$item = $head . "class $classname { $values }$tail";
 		}
 
-		$item = preg_replace_callback('/\b(1[2345]\d\d\d\d\d\d\d\d)\b(.*)/', function($m) { return $m[1] . $m[2] . " # " . date('Y-m-d H:i:s', $m[1]); }, $item);
+		$item = preg_replace_callback('/\b(1[2-9]\d\d\d\d\d\d\d\d)\b(.*)/', function($m) { return $m[1] . $m[2] . " # " . date('Y-m-d H:i:s', $m[1]); }, $item);
 		$item = preg_replace("#(=>?)\s*\n\s*(array|class)#", '$1 $2', $item); # array( and class on same line as key
 		$item = preg_replace_callback('#array \(\s+([^(){};]{1,100}),\s+\)#', function($m) { return "array (" . preg_replace("#\n\s+#", " ", $m[1]) . ")"; }, $item); # short arrays on 1 line
 		$item = preg_replace('#class (\S+) \{\s+([^({,;]+;)?\s+\}#', 'class $1 { $2 }', $item); # 1-element objects on 1 line
diff --git a/it_text.class b/it_text.class
index 660213e..f575344 100644
--- a/it_text.class
+++ b/it_text.class
@@ -57,18 +57,13 @@ function __construct($p = array())
 	# Read and merge texts from php files if none defined yet
 	foreach ($p['phpfiles'] as $phpfile)
 	{
-		$oldtext = $this->statictext;
-		if (is_array($ret = include($phpfile)))
+		$this->statictext += ($ret = include($phpfile));
+
+		if ($GLOBALS['debug_texts'])
 		{
-			$this->statictext += $ret;
-			if ($GLOBALS['debug_texts'])
-			{
-				$service = strpos($phpfile, $GLOBALS['ULTRAHOME']) !== false ? '' : it::match('/www/([^/.]+)', $phpfile);
-				$this->label_to_service += array_combine(array_keys($ret), array_fill(0, count($ret), $service));
-			}
+			$service = strpos($phpfile, $GLOBALS['ULTRAHOME']) !== false ? '' : it::match('/www/([^/.]+)', $phpfile);
+			$this->label_to_service += array_combine(array_keys($ret), array_fill(0, count($ret), $service));
 		}
-		else
-			$this->statictext = $oldtext + $this->statictext; # FIXME: compatibility mode
 	}
 
 	# Get array of supported languages and their names
@@ -274,18 +269,12 @@ function dump_php()
 
 	$oldmask = umask(002);
 	$filename = $this->p['phpfiles'][0];
-	$oldsyntax = file_exists($filename) && trim(it::exec('grep -q \'array(\' {filename} && echo array', ['filename' => $filename]));
-	if ((count($this->p['phpfiles']) == 1) && ($f = it::fopen($filename, 'w')))
+	if ((count($this->p['phpfiles']) == 1))
 	{
 		$dump = '<?php return ' . strtr(var_export($this->statictext, true), array("=> \n  array (" => "=> array(", "array (\n  '_'" => "array(\n'_'", "\n  ),\n  " => "\n),\n", "\n  ),\n" => "\n),\n", "\r" => "")) . ";\n?>\n";
-		if (!$oldsyntax)
-		{
-			it::file_put_contents($tmpfile = tempnam('/tmp/', 'texts_'), $dump);
-			$dump = it::exec('/www/server/bin/convertsyntax.php -a {tmpfile}', ['tmpfile' => $tmpfile]);
-			unlink($tmpfile);
-		}
-		$result = (fputs($f, $dump) !== false);
-		fclose($f);
+		it::file_put_contents($tmpfile = tempnam('/tmp/', 'texts_'), $dump);
+		$result = it::system('/www/server/bin/convertsyntax.php -a {tmpfile} >{filename}', ['tmpfile' => $tmpfile, 'filename' => $filename]) == 0;
+		unlink($tmpfile);
 	}
 
 	umask($oldmask);
diff --git a/it_url.class b/it_url.class
index 6b1f81c..dbd39e4 100644
--- a/it_url.class
+++ b/it_url.class
@@ -106,6 +106,7 @@ static function _postprocess($data, $p)
  * @param $p['files']         [fieldname => filename] of files to upload
  * @param $p['writefunction'] function to be called whenever data is received (for server-sent-events etc.)
  * @param $p['fetchsleep']    Number of seconds to wait after fetch, fractions ok
+ * @param $p['followlocation']Follow redirects [true]
  * @param $p['retries']       Number of retries if download fails, default 1
  * @param $p['retrysleep']    Number of seconds to wait before retry (additional to fetchsleep), fractions ok
  * @param $p['compression']   use compression (uses curl to do that)
@@ -170,7 +171,7 @@ function parse_http_header($header)
 static function _default_headers($url, $p)
 {
 	$search_subrequest = it::match('search\.ch/', $p['url']);
-	if ((!it::is_devel() || EDC('subreqcheck')) && $p['url'] && !$p['headers']['Accept-Language'] && T_lang() != T_defaultlang() && $search_subrequest && !it::match('\blogin\.|banner\.html|machines\.txt|mbtiles\.php|/itjs/|/images/|\.(de|fr|en|it)(\.js|\.html|\.txt|\.php|\.ics|\.pdf|\.json|\.csv|\.gif|\.jpg|\.png)', $p['url']))
+	if ((!it::is_devel() || EDC('subreqcheck')) && $p['url'] && !$p['headers']['Accept-Language'] && T_lang() != T_defaultlang() && $search_subrequest && !it::match('\blogin\.|banner\.html|machines\.txt|mbtiles\.php|/fonts/|/itjs/|/images/|\.(de|fr|en|it)(\.js|\.html|\.txt|\.php|\.ics|\.pdf|\.json|\.csv|\.gif|\.jpg|\.png)', $p['url']))
 		it::error(['title' => "Subrequest without language override", 'body' => [ $p ]]);
 
 	$headers = array_filter([
@@ -332,12 +333,13 @@ function request($p=array())
 /**
  * Get multiple URL in parallel with timeout. Needs to be called statically
  * @param $p parameter array with the following keys (same as it_url::get)
- * @param $p['urls']         array/generator of urls to get
- * @param $p['timeout']      timeout per read in seconds, defaults to 5. (TODO: fractions allowed?)
- * @param $p['totaltimeout'] timeout for the whole function call (fractions allowed)
- * @param $p['headers']      optional array of HTTP headers to send
- * @param $p['parallel']     max number of parallel requests
- * @param $p['noresults']    do not keep results around
+ * @param $p['urls']           array/generator of urls to get
+ * @param $p['timeout']        timeout per read in seconds, defaults to 5. (TODO: fractions allowed?)
+ * @param $p['totaltimeout']   timeout for the whole function call (fractions allowed)
+ * @param $p['followlocation'] follow redirects [true]
+ * @param $p['headers']        optional array of HTTP headers to send
+ * @param $p['parallel']       max number of parallel requests
+ * @param $p['noresults']      do not keep results around
  * @return array of contents (or false for errors like timesou) of resulting page using same
  *         keys as the urls input array, considering redirects, excluding headers
  */
@@ -361,13 +363,13 @@ static function get_multi($p=null)
 		curl_setopt($handle, CURLOPT_URL, it::replace([ '^//' => "http://" ], is_array($url) ? $url['url'] : $url));
 		curl_setopt_array($handle, $opts);
 		curl_multi_add_handle($mh, $handle);
-		$keys[$handle] = $key;
+		$keys[(int)$handle] = $key;
 		$handles[$key] = $handle;
 	};
 	$closehandle = function ($key) use (&$keys, &$handles, $mh) {
 		curl_multi_remove_handle($mh, $handles[$key]);
 		curl_close($handles[$key]);
-		unset($keys[$handles[$key]]);
+		unset($keys[(int)$handles[$key]]);
 		unset($handles[$key]);
 	};
 
@@ -406,7 +408,7 @@ static function get_multi($p=null)
 			{
 				if ($info['msg'] == CURLMSG_DONE)
 				{
-					$key = $keys[$info['handle']];
+					$key = $keys[(int)$info['handle']];
 					$content = curl_multi_getcontent($info['handle']);
 					if (isset($p['postprocess']))
 						$content = $p['postprocess']($content, ['it_error' => $retries[$key] < $p['retries'] ? false : (array)$p['it_error'] + ['title' => "invalid content from " . $urls[$key]]]);
diff --git a/it_user.class b/it_user.class
index 3bb9c3a..f05bb00 100644
--- a/it_user.class
+++ b/it_user.class
@@ -92,13 +92,6 @@ function read($id = null)
 	# Get username from database field (shortcut)
 	$this->username = $this->{$this->p['username_field']};
 
-	# username == uid means we don't have a username yet
-	if ($this->username && $this->username == $this->uid)	# FIXME 2020-12 CS Remove support for legacy user records with username = uid
-	{
-		it::error(['to' => 'schneider@search.ch', 'title' => "Legacy user record with username = uid ($this->uid)"]);
-		$this->username = "";
-	}
-
 	return $result;
 }
 
diff --git a/it_xml.class b/it_xml.class
index 2e687e1..93f8b4d 100644
--- a/it_xml.class
+++ b/it_xml.class
@@ -209,6 +209,9 @@ function end_element($dummy_parser, $name)
 
 	if (!$this->_stack[0]->consume($this->_p))
 	{
+		if (!isset($this->_stack[1]))
+			$this->_stack[1] = (object)[];
+
 		if (is_array($this->_stack[1]->$name))
 			array_push($this->_stack[1]->$name, $this->_stack[0]);
 		else if (isset($this->_stack[1]->$name))
diff --git a/test/getopt.t b/test/getopt.t
index aa2de52..48f7d9d 100755
--- a/test/getopt.t
+++ b/test/getopt.t
@@ -38,10 +38,9 @@ getopt_ok([], false, "Missing positional argument fails");
 getopt_ok(['posarg', '--argument'], false, "Missing long named argument fails");
 getopt_ok(['posarg', '-a'], false, "Missing short named argument fails");
 
-// FIXME 2020-10 NG enable after fixme in it.class
-// $GLOBALS['usage'] = it::replace('\s*\[VARARGS\]' => '', $GLOBALS['usage']);
-// getopt_ok(['posargs', 'vararg'], false, "Extra positional argument fails");
-// getopt_ok(['posargs', '--zero', 'vararg'], false, "Extra positional argument fails after long argument");
-// getopt_ok(['posargs', '-0', 'vararg'], false, "Extra positional argument fails after short argument");
-// getopt_ok(['posargs', '--argument', 'value', 'vararg'], false, "Extra positional argument fails after long argument with value");
-// getopt_ok(['posargs', '-a', 'value', 'vararg'], false, "Extra positional argument fails after short argument with value");
+$GLOBALS['usage'] = it::replace(['\s*\[VARARGS\]' => ''], $GLOBALS['usage']);
+getopt_ok(['posargs', 'vararg'], false, "Extra positional argument fails");
+getopt_ok(['posargs', '--zero', 'vararg'], false, "Extra positional argument fails after long argument");
+getopt_ok(['posargs', '-0', 'vararg'], false, "Extra positional argument fails after short argument");
+getopt_ok(['posargs', '--argument', 'value', 'vararg'], false, "Extra positional argument fails after long argument with value");
+getopt_ok(['posargs', '-a', 'value', 'vararg'], false, "Extra positional argument fails after short argument with value");
diff --git a/test/it.t b/test/it.t
index 64b0fac..a3abf32 100755
--- a/test/it.t
+++ b/test/it.t
@@ -12,7 +12,7 @@ $oldlocale = setlocale(LC_CTYPE, 0);
 ini_set('default_charset', 'utf-8');
 setlocale(LC_CTYPE, 'de_CH');		# required becuase we're checking German umlauts in latin1 mode
 
-function match($regex, $string, $expect, $name, $p = [])
+function _match($regex, $string, $expect, $name, $p = [])
 {
 	$GLOBALS['TEST_MORE_LEVEL'] = 1;
 	$pass = is (it::match($regex, $string, $p), $expect, $name);
@@ -24,103 +24,103 @@ function match($regex, $string, $expect, $name, $p = [])
 }
 
 
-match(
+_match(
 	'b', 'aaaabaaaa',
 	'b',
 	'simple regex'
 );
 
-match(
+_match(
 	'a/b', '   a/b   ',
 	'a/b',
 	'regex with /'
 );
 
-match(
+_match(
 	'aa(bb)aa(cc)aa(dd)qq', 'aabbaaccaaddqq',
 	['bb', 'cc', 'dd'],
 	'return array of captures'
 );
 
-match(
+_match(
 	'\bblah\b', ' blah ',
 	'blah',
 	'match \b at spaces'
 );
 
-match(
+_match(
 	'\bblah\b', 'blah',
 	'blah',
 	'match \b at end of string'
 );
 
-match(
+_match(
 	'\bblah\b', 'ablahc',
 	null,
 	'don\'t match \b at word chars'
 );
 
-match(
+_match(
 	'\bblah\b', 'Üblahä',
 	null,
 	'don\'t match \b at umlaute'
 );
 
-match(
+_match(
 	'\Bblah\B', ' blah ',
 	null,
 	'don\'t match \B at spaces'
 );
 
-match(
+_match(
 	'\Bblah\B', 'blah',
 	null,
 	'don\'t match \B at end of string'
 );
 
-match(
+_match(
 	'\Bblah\B', 'ablahc',
 	'blah',
 	'match \B at word chars'
 );
 
-match(
+_match(
 	'\Bblah\B', 'Üblahä',
 	'blah',
 	'match \B at umlaute'
 );
 
-match(
+_match(
 	'\w+', '  |#Üblahä   ',
 	'Üblahä',
 	'include umlaute in \w'
 );
 
-match(
+_match(
 	'[[:alpha:]]+', '  |#blahä   ',
 	'blahä',
 	'include umlaute in [[:alpha:]]'
 );
 
-match(
+_match(
 	'\W+', '  |#Üblahä  ',
 	'  |#',
 	'don\'t include umlaute in \W'
 );
 
-match(
+_match(
 	'\ba', 'äa',
 	null,
 	'\b must know umlauts'
 );
 
-match(
+_match(
 	'aaa\\\\w+', '   aaa\www  ',
 	'aaa\www',
 	'don\'t parse \w in \\\\w at beginning (match)'
 );
 
-match(
+_match(
 	'aaa\\\\w+', '   aaa\www  ',
 	'aaa\www',
 	'don\'t parse \w in \\\\w after chars (match)'
@@ -129,105 +129,105 @@ match(
 eval('$escapedwordregex = "' . it::convertregex('\w') . '";');
 $escapedwordregex = preg_replace('|[\\\\/]|', '', $escapedwordregex);
 
-match(
+_match(
 	'\\\\w+',  $escapedwordregex,
 	null,
 	'don\'t parse \w in \\\\w at beginning (no match)'
 );
 
-match(
+_match(
 	'aaa\\\\w+', 'aaa' . $escapedwordregex,
 	null,
 	'don\'t parse \w in \\\\w after chars (no match)'
 );
 
-match(
+_match(
 	'\\\\\\\\w+', '\\' . $escapedwordregex,
 	null,
 	'don\'t parse \w in \\\\\\\w (no match)'
 );
 
-match(
+_match(
 	'\\\\\\\\w+', '  \\\\www  ',
 	'\\\\www',
 	'don\'t parse \\\\\\\\w as \w (match)'
 );
 
-match(
+_match(
 	'[\w]+', '[[[]]]---',
 	null,
 	'replace \w in [\w] correctly (no match)'
 );
 
-match(
+_match(
 	'[\w]+', '  \\\\aword[[[]]]   ',
 	'aword',
 	'replace \w in [\w] correctly (match)'
 );
 
-match(
+_match(
 	'[\\\\w]+', ' blabergna ',
 	null,
 	'don\'t parse \w in [\\\\w] (no match)'
 );
 
-match(
+_match(
 	'[\\\\w]+', '  \\\\worda[[[]',
 	'\\\\w',
 	'don\'t parse \w in [\\\\w] (match)'
 );
 
-match(
+_match(
 	'[a\W]+', 'bbbbbbb a a%$+ accccc',
 	' a a%$+ a',
 	'\W in []'
 );
 
-match(
+_match(
 	'\\\\\\w+', '  \Üblahä  ',
 	'\Üblahä',
 	'parse \w in \\\\\\w at beginning'
 );
 
-match(
+_match(
 	'aaa\\\\\\w+', '  aaa\Üblahä  ',
 	'aaa\Üblahä',
 	'parse \w in \\\\\\w after chars'
 );
 
-match(
+_match(
 	'\w+', 'word1 wörd2 word_3',
 	['word1', 'wörd2', 'word_3'],
 	"test match_all function",
 	['all' => true]
 );
 
-match(
+_match(
 	'aBcD', '  aBcD  ',
 	'aBcD',
 	"caseinsensitive is default"
 );
 
-match(
+_match(
 	'\w+', 'Müller',
 	'Müller',
 	'\w matches umlaut in utf-8 mode'
 );
 
-match(
+_match(
 	'M.ller', 'Müller',
 	'Müller',
 	'. matches umlaut in utf-8 mode'
 );
 
-match(
+_match(
 	utf8_decode('ö'), utf8_decode('Ö'),
 	utf8_decode('Ö'),
 	'match umlaute in de_CH.latin1 case insensitive',
 	['utf8' => false]
 );
 
-match(
+_match(
 	utf8_decode('aöBÜ'), utf8_decode('AÖbü'),
 	utf8_decode('AÖbü'),
 	"match umlaute with non-utf-8 override in p",
@@ -235,35 +235,35 @@ match(
 );
 
 
-match(
+_match(
 	'abc', "aBc",
 	null,
 	"set case sensitivity by parameter",
 	['casesensitive' => 1]
 );
 
-match(
+_match(
 	'\w+', 'word1 wörd2 word_3',
 	['word1', 'wörd2', 'word_3'],
 	"test all => 1 without captures",
 	['all' => 1]
 );
 
-match(
+_match(
 	'\w+\s+(\d+)', 'word1 12 wörd2 3 word_3 4',
 	['12', '3', '4'],
 	"test all => 1 with one capture",
 	['all' => 1]
 );
 
-match(
+_match(
 	'(\w+)\s+(\d+)', 'word1 12 wörd2 3 word_3 4',
 	[['word1', '12'], ['wörd2', '3'], ['word_3', '4']],
 	"test all => 1 with captures",
 	['all' => 1]
 );
 
-match(
+_match(
 	'(\w+)\s+(\d+)', 'word1 12 wörd2 3 word_3 4',
 	[['word1', 'wörd2', 'word_3'], ['12', '3', '4']],
 	"test all => 1,pattern_order => 1",
@@ -271,7 +271,7 @@ match(
 );
 
 ini_set('default_charset', 'iso-8859-1');
-match(
+_match(
 	'aöBÜ', "AÖbü",
 	'AÖbü',
 	"match utf-8 umlaute in case insensitive mode with utf8 override",
@@ -381,6 +381,10 @@ is(it::ucwords('foo bär über'), 'Foo Bär Über');
 # it::substr_replace
 is(it::substr_replace('abcdefgh', 'xyz', 2, 4), substr_replace('abcdefgh', 'xyz', 2, 4), 'it::substr_replace the same as substr_replace for ascii');
 is(it::substr_replace('✔☯♥', '☃☃', 1, 1), '✔☃☃♥', 'it::substr_replace for utf-8');
+is(it::substr_replace('', 'xyz', 0, 0), substr_replace('', 'xyz', 0, 0), 'it::substr_replace with empty haystack the same as substr_replace');
+is(it::substr_replace('abc', 'xyz', 0, 2), substr_replace('abc', 'xyz', 0, 2), 'it::substr_replace replacing to end of haystack');
+is(it::substr_replace('abc', 'xyz', 0, 10), substr_replace('abc', 'xyz', 0, 10), 'it::substr_replace replacing past end of haystack');
+is(it::substr_replace('abcdefgh', 'xyz', 10, 4), substr_replace('abcdefgh', 'xyz', 10, 4), 'it::substr_replace outside of string');
 
 is(grapheme_strlen("\xc1"), null, "need grapheme_strlen side effect for any2utf8");
 
@@ -448,6 +452,7 @@ is(it::mod(7, 4), 3);
 
 is(it::map('5*$k+$v', [0 => 1, 1 => 2]), [1, 7]);
 is(it::map(function($k, $v) {return 5*$k+$v;}, [0 => 1, 1 => 2]), [1, 7]);
+is(it::map(function($v) {return 2*$v;}, [0 => 1, 1 => 2]), [2, 4]);
 is(it::map('strlen', ["aaa", "aa"]), [3, 2]);
 is(it::map('it::ucfirst', ["aaa"]), ["Aaa"]);
 $dom = new DOMDocument;
@@ -465,6 +470,8 @@ is(it::map('2*$v', ['foo' => 1, 'bar' => 2], ['keys' => 'foo']), ['foo' => 2, 'b
 # it::filter
 is(it::filter('$v > 2', [1, 5 => 2, 2 => 3]), [2 => 3]);
 is(it::filter('$k > 2', [1, 5 => 2, 2 => 3]), [5 => 2]);
+is(it::filter(function($v) {return $v > 2;}, [1, 5 => 2, 2 => 3]), [2 => 3]);
+is(it::filter(function($k, $v) {return $k > 2;}, [1, 5 => 2, 2 => 3]), [5 => 2]);
 
 is(it::split("b", "aba"),                               ["a", "a"]);
 is(it::split("b", "aBa"),                               ["a", "a"]);
diff --git a/test/it_text.t b/test/it_text.t
index 5629d51..6726996 100755
--- a/test/it_text.t
+++ b/test/it_text.t
@@ -1,7 +1,9 @@
 #!/www/server/bin/php
 <?php
 
+$obj = (object)[];
 $obj->x = "attr";
+$obj->y = (object)[];
 $obj->y->z = "attr";
 
 is(it_text::transmogrify(""), "");
diff --git a/test/it_xml.t b/test/it_xml.t
index 27f910f..e21f052 100755
--- a/test/it_xml.t
+++ b/test/it_xml.t
@@ -3,7 +3,7 @@
 
 # Tests for xml.class
 
-function match($xmldata, $expected, $name, $prefix = "", $p = [])
+function _match($xmldata, $expected, $name, $prefix = "", $p = [])
 {
 	$classname = ($prefix ?: "it") . "_xml";
 	$varname  = $prefix . "foo";
@@ -31,37 +31,37 @@ function match($xmldata, $expected, $name, $prefix = "", $p = [])
 	);
 }
 
-match(
+_match(
 	'<foo />',
 	'foo Object ( ) ',
 	'empty tag'
 );
 
-match(
+_match(
 	'<foo /><foo />',
 	'Array ( [0] => foo Object ( ) [1] => foo Object ( ) ) ',
 	'multiple empty tags converted to array'
 );
 
-match(
+_match(
 	'<foo title="Zürich">Stüssihofstadt</foo>',
 	'foo Object ( [attr] => Array ( [title] => Zürich ) [val] => Stüssihofstadt ) ',
 	'simple tag with latin1 content and attribute'
 );
 
-match(
+_match(
 	'<foo><ns:a.b.-c ns2:d.e-f="value" /></foo>',
 	'foo Object ( [a_b__c] => a_b__c Object ( [attr] => Array ( [d_e_f] => value ) ) ) ',
 	'Tags and attributes with name space and special characters'
 );
 
-match(
+_match(
 	'<foo>x &amp; y</foo>',
 	'foo Object ( [val] => x & y ) ',
 	'Character data with entities'
 );
 
-match(
+_match(
 	'<foo>x &uuml; y</foo>',
 	utf8_decode('foo Object ( [val] => x ü y ) '),
 	'Manual encoding override',
@@ -69,13 +69,13 @@ match(
 	['encoding' => "iso-8859-1"]
 );
 
-match(
+_match(
 	'<foo>&amp;amp; &lt;a&gt; &#38;amp; &#60;b&#62; &#x26;amp; &#x3C;c&#x3E; &uuml;</foo>',
 	'foo Object ( [val] => &amp; <a> &amp; <b> &amp; <c> ü ) ',
 	'Predecode illegal entities while keeping properly encoded ones'
 );
 
-match(
+_match(
 	'<foo>&amp;amp; &lt;a&gt; &#38;amp; &#60;b&#62; &#x26;amp; &#x3C;c&#x3E; &#xFC;</foo>',
 	utf8_decode('foo Object ( [val] => &amp; <a> &amp; <b> &amp; <c> ü ) '),
 	'Predecode illegal entities while keeping properly encoded ones (iso-8859-1)',
@@ -83,7 +83,7 @@ match(
 	['encoding' => "iso-8859-1"]
 );
 
-match(
+_match(
 	"<foo>a\x05b</foo>",
 	'foo Object ( [val] => a b ) ',
 	'Illegal latin 1 character',
@@ -109,7 +109,7 @@ function __construct($xmldata)
 
 }
 
-match(
+_match(
 	'<myfoo />',
 	'myfoo Object ( [inheritbaseclass] => ) ',
 	'Inheritance and constructor (critical for e.g. tel_xmlentry)',
@@ -118,7 +118,7 @@ match(
 
 $x = new foo("<foo></foo>", ['prefix' => "test"]);
 $x->set(['gna' => 42, 'bar' => ['baz' => ["qux", "quux"]]]);
-match(
+_match(
 	$x->to_xml(),
 	'foo Object ( [gna] => gna Object ( [val] => 42 ) [bar] => bar Object ( [baz] => Array ( [0] => baz Object ( [val] => qux ) [1] => baz Object ( [val] => quux ) ) ) ) ',
 	"Method set()"