<?php
/*
**	Copyright (C) 1995-2021 by the ITools Authors.
**	This file is part of ITools - the Internet Tools Library
**
**	ITools is free software; you can redistribute it and/or modify
**	it under the terms of the GNU General Public License as published by
**	the Free Software Foundation; either version 3 of the License, or
**	(at your option) any later version.
**
**	ITools is distributed in the hope that it will be useful,
**	but WITHOUT ANY WARRANTY; without even the implied warranty of
**	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**	GNU General Public License for more details.
**
**	You should have received a copy of the GNU General Public License
**	along with this program.  If not, see <http://www.gnu.org/licenses/>.
**
**	it.class - static functions
*/

class it
{
static $error_context; # Callback, can provide $p params like 'head' and 'toscreen'

/**
 * Create config class with static members initialized (e.g. $home).
 * NOTE: PHP5 ONLY
 * @param $p Static members to be generated in newly created class
 *      service: Class name of created class (default: from caller path)
 *      home: Home directory of application (default: from caller path)
 *      site: Domain name of application (default: from caller path)
 *      db: Database of application (default: from caller path)
 */
static function createconfig($p = array())
{
	$stack = debug_backtrace();
	$filename = $stack[0]['file'];
	preg_match('!/www/((\w+)[^/]+)!', $filename, $parts);

	$p += array(
		'home' => $parts[0],
		'site' => $parts[1],
		'db' => strtr($parts[1], ".-", "__"),
		'service' => $parts[2],
	);

	if (class_exists($p['service'] . "_tools"))
		$extends = "extends {$p['service']}_tools ";

	$code = array("class {$p['service']} $extends{");

	foreach ($p as $name => $value)
		$code[] = "static \$$name = " . var_export($value, true) . ";";

	$code[] = "}";
	eval(implode("\n", $code));
}


/**
 * Clone an object and return copy, works for all PHP versions
 */
static function &cloneobj(&$object)
{
	$result = (is_object($object) && version_compare(zend_version(), 2, '>=')) ? clone($object) : $object;

	return $result;	# PHP internals need a tmp var to return by ref
}


/**
 * Append all arguments to a logfile (tab separated). Date will be added to filename and line
 * @param $name Name of logfile. Will be in log/ of service unless it starts with /
 * @param $args Varargs to log, will be tab separated.
 */
static function log($name, ...$args)
{
	$line = date("Y-m-d H:i:s") . "\t" . implode("\t", $args);
	self::log_line($name, $line);
	return $line;
}

/**
 * Append line to a logfile. Date will be added to filename
 * @param $name Name of logfile. Will be in log/ of service unless it starts with /
 * @param $line Line to log.
 */
static function log_line($name, $line)
{
	$basefn = substr($name, 0, 1) == "/" ? $name : ($GLOBALS['ULTRAHOME'] == "/opt/ultra/bin" ? it::match('/www/[^/]*', getcwd()) : $GLOBALS['ULTRAHOME']) . "/log/$name";
	$fullfn = $basefn . "-" . date('Ymd');

	if (substr($fullfn, 0, 1) == "/")
	{
		if (!file_exists($fullfn))
		{
			$tmp = getmypid();
			@touch("$fullfn.$tmp");
			@chgrp("$fullfn.$tmp", "www");
			@chmod("$fullfn.$tmp", 0664);
			@rename("$fullfn.$tmp", $fullfn);
			@unlink($basefn);
			if (@readlink($basefn) < basename($fullfn)) # may be older due to $GLOBALS['debug_time']
				@symlink($fullfn, $basefn);
		}

		it::file_put_contents($fullfn, $line . "\n", FILE_APPEND | LOCK_EX);
	}
}


/**
 * Store timings for appending to log/timer_log-* in auto_append.php
 */
static function timerlog($label = '')
{
	if ($GLOBALS['debug_timerlog'])
		$GLOBALS['ULTRATIMERLOG'] .= sprintf(" %s:%d", $label, (gettimeofday(true) - $GLOBALS['ULTRATIME'])*1000);
}


/**
 * If display_errors is on or stdout is a tty, shows error in page or on stdout respectively
 * If display_errors is off, mails (with rate limiting) diagnostics to .diffnotice addresses or file owner or SERVER_ADMIN
 *
 * Controlling error mail content
 *  @param $p either error title or assoc array of params, see below
 *  @param $p['title'] error title, one line. also accepted in $p[0] (with priority). false means suppress error
 *  @param $p['body'] error body: multiline string or any data type (will be dumped)
 *  @param $p['backtraceskip'] number of stack levels to drop
 *  @param $p['skipfiles'] files to skip in backtrace
 *  @param $p['omitdebuginfo'] value 1 omits long stack and var dumps, value 2 also minimal infos
 *
 * Controlling the sending of mails
 *  @param $p['to'] comma separated recipient list
 *  @param $p['timewindow'] in secs. "5-35" means for an mail to be sent, another err must occur 5 to 35 seconds after first one
 *  @param $p['id'] id of error, used with timewindow, defaults to file and line of caller
 *  @param $p['blockmailid'] block mail for $p['blockmail'] seconds with same id. Default: $p['to']
 *  @param $p['blockmail'] number of seconds to block mails after having sent a mail [3600]
 *  @param $p['okstate'] give current ok label and ok state, e.g. telresult=1 for working. see failcount
 *  @param $p['failcount'] give number of consecutive okstate failures needed for creating an error [2]
 *  @param $p['fatal'] exit after displaying error
 *  @param $p['graceperiod'] in seconds. DEPRECATED, see $p['timewindow']
 *  Use debug param 'verboseerrors' to include $p['body'] when interactive
 *
 * @return always null (so users can return it::error() in error cases)
 *
 * TIMEWINDOW 5-35 (x = error occurs, no mail; m = error occurs, mail sent)
 *   time        ---|5 secs|30 secs      |------|5 secs|30 secs
 *                                                          |5 secs|30 secs
 *                  x  x                        x   x       m   x     m   x
 * BLOCKMAIL
 *   time        ---|blockmail|--|blockmail|---->
 *   blockmailid    m  x     x   m   x
 *
 * GRACPERIOD/TIMEWINDOW (DEPRECATED, graceperiod is first argument of 5-35, timewindow is difference between 5 and 35)
 *   time        ---|graceperiod|timewindow|
 *                                   |graceperiod|timewindow|--.
 *                                                             |graceperiod|timewindow|------>
 *   id             x  x     x       m    x   x                x     x
 */
static function error($p = array(), $extra = null)
{
	$p = $origp = (array)$p;

	if (error_reporting() == @error_reporting() || $p[0] === false || $p['title'] === false) # called with @ or suppressed
		return $p['fatal'] ? self::_exit($p) : null;

	$p['title'] = $p[0] ?: $p['title']; # handle 'it_error' => "oops" that was cast to array on the way
	$p['title'] = grapheme_substr($p['title'], 0, 2000) ?: substr($p['title'], 0, 2000);
	$p += is_callable(self::$error_context) ? (self::$error_context)() : [];

	$home = $GLOBALS['ULTRAHOME'];

	# support for errlike() in tests
	$GLOBALS['ULTRAERROR'] = $p['title'];
	if ($GLOBALS['ULTRANOERRORS'])
		return null;

	if ($extra)
		$p = ['title' => 'extraneous params passed to it::error', 'fatal' => $p['fatal']];

	foreach (array_slice(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 10), 1) as $level)
		if ($level["class"] . $level["type"] . $level["function"] == "it::error")
			return null; # prevent recursion

	if ($_SERVER['REMOTE_ADDR'])
		$url = "http://" . it::replace([':443$' => ""], $_SERVER['HTTP_HOST']) . $_SERVER['REQUEST_URI']; # we ignore https for easier debugging
	else
		$url = $_SERVER['SCRIPT_NAME'] . " " . implode(" ", array_slice($GLOBALS['argv'], 1)) . " (pwd " . $_SERVER['PWD'] . ")";

	$authors = it::replace(['\*' => ""], join(", ", it::grep('\*', (array)@it::cat("$home/.diffnotice")->lines, ['invert' => true])));
	if (!$p['to'])
		unset($p['to']); # allow defaults to kick in
	$p += array(
		'title' => "it::error",
		'to' => $authors ? $authors : (get_current_user() ? get_current_user() : $_SERVER['SERVER_ADMIN']),
		'graceperiod' => 60,
		'timewindow' => 25*3600,
		'blockmail' => $p['okstate'] ? 60 : 3600,
		'omitdebuginfo' => false,
		'failcount' => 2,
		'id' => $p['timewindow'] ? "it_error_id_" . it_debug::backtrace(['skipfiles' => "it.class", 'levels' => 1]) : 0,
	);

	if (it::match('-', $p['timewindow']) && ($parts = explode("-", $p['timewindow'])))
		list($p['graceperiod'], $p['timewindow']) = [$parts[0], $parts[1] - $parts[0]];
	$p += array('blockmailid' => $GLOBALS['ULTRASITE'] . "." . md5($p['to']));
	$trace = it_debug::backtrace(array('skiplevels' => $p['backtraceskip'], 'skipfiles' => $p['skipfiles'], 'trace' => $p['trace'])); # moved in here for performance in mass error case

	@mkdir("/tmp/alertdata");
	@chmod("/tmp/alertdata", 0777);

	if ($p['okstate'])
	{
		list($okfn, $okstatus) = explode("=", "$home/okstate_" . $p['okstate']);
		$failcount = $okstatus ? 0 : (int)@file_get_contents($okfn) + 1; # NOPHPLINT
		file_put_contents($okfn, "$failcount\n"); # NOPHPLINT
		if ($failcount != $p['failcount'] && $failcount != $p['failcount'] * 30)
			return $p['fatal'] && !$okstatus ? self::_exit($p) : null; # nothing to report
	}

	$toscreen = getenv('NOERRORMAILS') || ini_get('display_errors') || (defined("STDOUT") && posix_isatty(STDERR)) || EDC('astwin') || EDC('asdevel') || $p['toscreen'];

	if ($toscreen && !it::is_live())
		$GLOBALS['debug_noredir'] = 1;

	if (!$toscreen) # this error can only be sent by mail: find out if we want to suppress it
	{
		if (!$p['id'])
			$sendmail = true;
		else
		{
			$errstampfn = "/tmp/alertdata/errstamp_" . urlencode($p['id']);
			$errstamp = @filemtime($errstampfn);
			$errstampage = time() - $errstamp;
			$sendmail = $errstampage >= $p['graceperiod'] && $errstampage <= $p['graceperiod'] + $p['timewindow'];
			if ($errstampage > $p['graceperiod'])
			{
				@unlink($errstampfn);
				@touch($errstampfn);
			}
		}

		if ($sendmail)
		{
			$lastsentfn = "/tmp/alertdata/it_error_mailsent_" . urlencode($p['blockmailid']);
			clearstatcache();
			$sendmail = time() - max(@filemtime($lastsentfn), it_cache::get($lastsentfn, ['distributed' => true])) > $p['blockmail'] && !it_cache::get('it_error_mute_' . $GLOBALS['ULTRASITE'], ['distributed' => true]);
			if ($sendmail)
			{
				@unlink($lastsentfn);
				@touch($lastsentfn);
				if (it::is_live())
					@it_cache::put($lastsentfn, time(), ['distributed' => true, 'ttl' => 7*86400, 'safety' => 0]);
			}
		}
	}

	if ($p['fatal'] && $_SERVER['REMOTE_ADDR'])
	{
		http_response_code(500);
		echo "<!DOCTYPE html><html><head><title></title></head><body>Internal Server Error</body></html>\n";
	}

	if ($toscreen || $sendmail)
	{
		$p['body'] = is_string($p['body']) || !array_key_exists('body', $p) ? $p['body'] : it_debug::dump([$p['body']], ['html' => false, 'short' => true, 'color' => false]);
		if (strlen($p['body']) > 500000 || it::match('[\x00-\x08\x0E-\x1F]', $p['body'], ['utf8' => false]) !== null)
		{
			file_put_contents($datafn = "$home/error-" . substr(md5($p['body']), 0, 2), $p['body']); # NOPHPLINT
			$p['body'] = "Body: " . getenv('HOSTNAME') . ":$datafn";
		}

		$body =
			($p['omitdebuginfo'] >= 2 ? "" : ($url && !$toscreen? "Title:  {$p['title']}\nUrl:    $url\n" : "") .
			($trace ? ($sendmail ? "" : "  ") . "Trace:  $trace\n" : "")) .
			(!$sendmail || $p['omitdebuginfo'] >= 2 ? "" : "Host:   " . getenv('HOSTNAME') . " at " . date("Y-m-d H:i:s") . (($t = time() - $_SERVER['REQUEST_TIME']) ? " (invoked {$t}s before)" : "") . "\n") . # no it::date() due to time- debug param
			($sendmail ? $p['head'] : "") .
			($p['id'] && !$toscreen ? "Filter: timewindow=" . $p['graceperiod'] . "-" . ($p['graceperiod'] + $p['timewindow']) . "  (previous err: " . it::date('', $errstamp) . ")\n" : "") .
			(!$origp['blockmail'] || $p['omitdebuginfo'] || $toscreen ? "" : "Block-resend: " . $origp['blockmail'] . " seconds\n") .
			($p['body'] ? ($p['omitdebuginfo'] ? "" : "Body:\n") . trim($p['body'])."\n\n" : "");

		if ($sendmail || EDC('verboseerrors')) # we're mailing: send maximum info
		{
			$p['title'] = it::replace(['alert:|server:|^: "' => "", '(pw|passw|password\d*|secret)(=)[^&\s]*' => '$1$2*****'], it::match('[^.]*', $GLOBALS['ULTRASITE']) . ": " . $p['title']) . " (via " . getenv('HOSTNAME') . ")";

			if (!$p['omitdebuginfo'])
			{
				$backtrace = $p['trace'] ?: array_slice(debug_backtrace(0), $p['backtraceskip']);
				$mediumstack = it_debug::backtrace(['trace' => $backtrace, 'format' => "medium"]);
				$longstack   = it_debug::backtrace(['trace' => $backtrace, 'format' => "long"]);
				$reqbody = it::file_get_contents("php://input");

				$body .= ($trace and $t = it::exec('grep -h {0} `ls 2>/dev/null {1}/log/alertlog-*|tail -3` /dev/null 2>/dev/null | grep ^2 | cut -d : -f 1-2 | sort | uniq -c | tail -10', $trace, $home)) ? "Histogram: (last 10 affected minutes in 3 days)\n$t\n" : "";
				$body .= $mediumstack ? "Stack:\n  " . it::replace(["\n" => "\n  "], $mediumstack) . "\n" : "";
				$body .= $_GET        ? "\$_GET: "    . var_export($_GET, true) . "\n\n" : "";
				$body .= $_POST       ? "\$_POST: "   . var_export($_POST, true) . "\n\n" : "";
				$body .= $reqbody     ? "\$reqbody: " . var_export($reqbody, true) . "\n\n" : "";
				$body .= $_COOKIE     ? "\$_COOKIE: " . var_export($_COOKIE, true) . "\n\n" : "";
				$body .= $_SERVER['REMOTE_ADDR'] ? "" : "Pstree:\n" . it::exec("pstree -als " . getmypid() . " | head -n -3") . "\n";
				$body .= $_SERVER     ? "\$_SERVER: " . var_export($_SERVER, true) . "\n\n" : "";
				$body .= $_FILES      ? "\$_FILES:  " . var_export($_FILES, true) . "\n\n" : "";
				$body .= "Processes:\n" . it::exec('ps auxf | egrep -v "rotatelogs|getbanner|logaction|httpd|systemd|sd-pam"|egrep "^www|^cron"') . "\n";
				$body .= $longstack  ? "Full stack: "     . "$longstack\n" : "";
				$body  = it::replace(array('(pw|passw|password\d*|secret)(\' => |\] => |=)[^&\s]*' => '$1$2********'), $body, array('utf8' => false));
			}

			$type = ($p['fatal'] ? (it::is_live() ? "FATAL: " : "Fatal: ") : "Error: ");
			$debugparams = it::match('\.([^-.]+)', $_SERVER['HTTP_HOST'], ['all' => true]);
			if (!it::is_live() || array_diff($debugparams, it::match('[-\w]+', $home, ['all' => true]), ["devel", "twin", gethostname()]))
				$type = mb_strtolower($type);

			$user = posix_getpwuid(posix_geteuid())['name'];

			if ($sendmail)
				 it::mail([
					 'From' => "\"$user@" . gethostname() . "\" <$user>",
					 'To' => $p['to'],
					 'Reply-To' => $p['to'],
					 'Cc' => $GLOBALS['it_defaultconfig']['error_cc'],
					 'Subject' => $type . substr($p['title'], 0, 160),
					 'Body' => it::replace(['\x00' => "[NULLBYTE]"], $body, ['utf8' => false]),
					 'forcemail' => !it::is_devel(),
				 ]);
			else
				echo $body;

			 $p['title'] = "Mail: " . $p['title'];
		}
		else if ($_SERVER['REMOTE_ADDR'] && EDC('edplain')) # toscreen mode: web
			echo $p['title'] . "\n";
		else if ($_SERVER['REMOTE_ADDR']) # toscreen mode: web
			echo "<pre style='z-index:10000; position:relative; background:white'>" . htmlspecialchars($p['title'] . "\n" . rtrim($body), ENT_COMPAT, "iso-8859-1") . "</pre>"; # works with iso-8859-1 or utf-8, UTF8SAFE
		else  # toscreen mode: shell (outputs to stderr)
			error_log(substr($p['title'], 0, 100000) . ($p['omitdebuginfo'] >= 2 ? "" : " in " . ($trace ? $trace : " {$p['file']}:{$p['line']}") . " Url: $url " . (EDC('verboseerrors') ? D($p['body']) : "")));
	}

	if ($_SERVER['REMOTE_ADDR']) # additional entry in log/error_log
		error_log("it::error: " . $p['title'] . " Url: $url");

	it::log("alertlog", $p['title'] . " in " . ($trace ? $trace : "{$p['file']}:{$p['line']}") . " Url: $url" . ($p['body'] ? "\n" . substr(D($p['body']), 0, 5000) . "\n" : ""));

	return $p['fatal'] ? self::_exit($p) : null;
}


/**
 * Same as it::error(), plus exit (unless okstate given and true)
 * @see error()
 */
static function fatal($p)
{
	it::error(['fatal' => true, 'backtraceskip' => 2] + (array)$p);
}


/**
 * Print message to stderr and exit with error code
 */
static function bail($message = "Bailed.")
{
	fputs(STDERR, rtrim($message, "\n") . "\n");
	exit(1);
}

static function _exit($p)
{
	if ($GLOBALS['it_defaultconfig']['fatal_throws_exception'])
		throw new Exception($p['title']);
	exit(99);
}


/**
 * Check wether I am on a live server for this service. Honors 'aslive' and 'asdevel' debug vars
 * @return true if servertype is 'live'
 */
static function is_live()
{
	return it::servertype('live');
}


/**
 * Check wether I am on a development server for this service. Honors 'aslive' and 'asdevel' debug vars
 * @return true if servertype is 'devel'
 */
static function is_devel()
{
	return it::servertype('devel');
}

/**
 * Check if running on given server type(s)
 */
static function servertype($pattern)
{
	return it::match($pattern, $GLOBALS['debug_aslive'] ? 'live' : ($GLOBALS['debug_asdevel'] ? 'devel' : ($GLOBALS['ULTRASERVERTYPE'] ?: 'live')));
}


/**
 * Check whether an IP adress lies within a given range. Supports IPv4 and IPv6
 * @param $ip IP address (192.168.42.123)
 * @param $cidrs IP range in CIDR notation (192.168.42.64/26) or array of ranges
 * @return true if $ip is within $cidr
 */
static function cidr_match($ip, $cidrs)
{
	foreach ((array)$cidrs as $cidr)
	{
		list($subnet, $mask) = explode('/', $cidr);
		$ip_bin = inet_pton($ip);
		$subnet_bin = inet_pton($subnet);
		$valid_bytes = $mask ? $mask >> 3 : 42;
		$bitmask = 256 - (1 << (8 - ($mask & 7)));
		$lastbyte_matched = $bitmask ? (ord($ip_bin[$valid_bytes]) & $bitmask) == (ord($subnet_bin[$valid_bytes]) & $bitmask) : true;

		if (substr($ip_bin, 0, $valid_bytes) == substr($subnet_bin, 0, $valid_bytes) && $lastbyte_matched)
			return true;
	}

	return false;
}


/**
 * check whether an IP address is a private, loopback, link-local or reserved
 * address. Supports IPv4 and IPv6
 * @param $host hostname or IP address as string
 * @return true if $host is in a private, loopback, link-local or reserved
 *         network block
 */
static function is_private_ip($host)
{
	if ($host == filter_var($host, FILTER_VALIDATE_IP))
		$ips = [$host];
	else
	{
		$ips = [];
		foreach ([DNS_A, DNS_AAAA] as $type)
		{
			$records = false;
			for ($try = 0; $try < 3 && $records === false; $try++)
				foreach ((array)($records = @dns_get_record($host, $type)) as $record)
					$ips[] = $record[$type == DNS_A ? 'ip' : 'ipv6'];
		}
	}

	foreach ($ips as $ip)
	{
		if ($ip !== filter_var($ip, FILTER_VALIDATE_IP))
			continue;
		$gotvalidip = true;
		if ($ip !== filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE))
			return true;
	}

	return $gotvalidip ? false : true;
}


/**
 * Convert regex for preg (adds and escapes delimiter, adds modifiers)
 * @param $pattern Regex to convert
 * @param $p['casesensitive'] Regex is case sensitive (omit modifier i)
 * @param $p['multiline'] add modifier m: ^ and $ match \n
 * @param $p['singleline'] add modifier s: . matches \n
 * @param $p['utf8'] add modifier u. This is the default if default_charset is utf-8, override with $p['utf8'] = false
 * @param $p['extended'] add modifier x (non signifcant whitespace)
 * @return converted regex to use with preg
 */
static function convertregex($pattern, $p = null)
{
	return '/' . strtr($pattern, array('/' => '\/')) . '/' .
		(!$p['casesensitive'] ? 'i' : '') .
		($p['multiline'] ? 'm' : '') .
		($p['singleline'] ? 's' : '') .
		($p['extended'] ? 'x' : '') .
		((!isset($p['utf8']) && ini_get('default_charset') == 'utf-8' || $p['utf8']) ? 'u' : '');
}


/**
 * Try to match string against regex. Case insensitive by default.
 * @param $pattern Regex to match against
 * @param $string String to match
 * @param $p['offset_capture'] Set flag preg_offset_capture (returns offsets with the matches).
 * @param $p['all'] Return every match as array instead of first match.
 * @param $p contains pattern modifiers, @see convertregex()
 * @return Matched string or null
 */
static function match($pattern, $string, $p = null)
{
	if (!preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p)
		$r = preg_match('!' . $pattern . '!i' . (ini_get('default_charset') == 'utf-8' ? 'u' : ''), $string, $m); # fast path for simple patterns
	else
	{
		$flags = $p['offset_capture'] ? PREG_OFFSET_CAPTURE : 0;

		if ($p['all'])
			$r = preg_match_all(it::convertregex($pattern, $p), $string, $m, $flags | PREG_PATTERN_ORDER, $p['offset']);
		else
			$r = preg_match(it::convertregex($pattern, $p), $string, $m, $flags, $p['offset']);
	}

	if (!$r)	# no match
	{
		if (preg_last_error() == PREG_BACKTRACK_LIMIT_ERROR)
			it::error("Exceeded pcre.backtrack_limit of " . ini_get('pcre.backtrack_limit') . " bytes");
		else if (preg_last_error() == PREG_BAD_UTF8_ERROR)
			it::error("Invalid utf-8 in it::match haystack: " . substr($string, 0, 500)); # UTF8SAFE

		$result = $p['all'] ? array() : null;
	}
	else if (count($m) == 1)	# no capture
		$result = $m[0];
	else if (count($m) == 2)	# one capture
		$result = $m[1];
	else if ($p['all'] && !$p['pattern_order'])	# captures, reorder pattern_order to set_order but without first element
		$result = array_map(null, ...array_slice($m, 1));
	else	# captures, don't return first element (matched string)
		$result = array_slice($m, 1);

	return $result;
}


/**
 * Replace parts of a string matched by a pattern with according replacement string. See convertregex for named parameters.
 * @param $replacementes Array with patterns as keys and replacement strings as values
 * @param $string String to change
 * @param $p['limit'] limit number of replacements (default: all)
 * @return New string
 * @see convertregex for more options in $p
 */
static function replace($replacements, $string, $p = array())
{
	$encoding = ini_get('default_charset') == 'utf-8' ? 'u' : '';
	foreach ($replacements as $pattern => $dummy)
		$patterns[] = !preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p ? "!$pattern!i$encoding" : it::convertregex($pattern, $p);

	$result = preg_replace($patterns, $replacements, $string, isset($p['limit']) ? $p['limit'] : -1);

	 if ($result === null && preg_last_error() == PREG_BAD_UTF8_ERROR)
		it::error("Invalid utf-8 in it::replace haystack: " . substr($string, 0, 500)); # UTF8SAFE

	return $result;
}

/**
 * Returns only the array elements matching the given regex
 * @param $pattern     Regex to match against
 * @param $array       array to grep
 * @param $p           parameters for it::match or 'invert' => true to return non-matching lines
 * @return New array
 */
static function grep($pattern, $array, $p = array())
{
	if (!preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p)
		$result = preg_grep('!' . $pattern . '!i' . (ini_get('default_charset') == 'utf-8' ? 'u' : ''), $array, $p['invert'] ? PREG_GREP_INVERT : 0); # fast path
	else
		$result = preg_grep(it::convertregex($pattern, $p), $array, $p['invert'] ? PREG_GREP_INVERT : 0);

	return $result;
}


/**
 * Returns string split by pattern (like preg_split but utf8 and case insensitive)
 * @param $pattern Regex to match against, no delimiters
 * @param $subject String to split
 * @param $p['limit'], $p['no_empty'], $p['delim_capture'], $p['offset_capture'] See preg_split()
 */
static function split($pattern, $subject, $p = array())
{
	$flags = ($p['no_empty'] ? PREG_SPLIT_NO_EMPTY : 0) | ($p['delim_capture'] ? PREG_SPLIT_DELIM_CAPTURE : 0) | ($p['offset_capture'] ? PREG_SPLIT_OFFSET_CAPTURE : 0);

	return preg_split(it::convertregex($pattern, $p), $subject, $p['limit'] ?: -1, $flags);
}

/**
 * Convert string or array to utf8 if it was not already utf-8 before. Also handles double encoding
 * @param $value String or array to convert
 * @param $errprefix Error message to output if anything needed to be done
 * @return Same string in utf-8 encoding
 */
static function any2utf8($value, $errprefix = "")
{
	if (is_array($value))
	{
		foreach ($value as $idx => $v)
			$newarr[self::any2utf8($idx)] = is_string($v) || is_array($v) ? self::any2utf8($v, $errprefix) : $v;
		$value = (array)$newarr;
	}
	else if (is_string($value))
	{
		if (grapheme_strlen($value) === null)
			list($value, $error) = array(it::utf8_encode($value), it::utf8_encode("incorrect utf8-encoding. input=$value"));
		if (preg_match('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', $value))
			list($value, $error) = array(it::any2utf8(preg_replace_callback('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', function($m) {return it::utf8_decode($m[0]);}, $value)), $errprefix ? "double utf8-encoding. input=$value" : "");
		if (preg_match('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', $value))
			list($value, $error) = array(preg_replace('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', " ", $value), "forbidden utf-8 character. input=$value");
		$value = preg_replace('/\xc2\xad/', '', $value);	# Kill invisible soft hyphens
		$value = normalizer_normalize($value, Normalizer::FORM_C);
		if ($error && $errprefix)
			it::error(array('title' => "$errprefix: " . trim($error)));
	}

	return $value;
}

/**
 * Uppercase first character similar to ucfirst() but for mbstring.internal_encoding
 */
static function ucfirst($string)
{
	return mb_strtoupper(mb_substr($string, 0, 1)) . mb_substr($string, 1);
}

/**
 * Lowercase first character similar to lcfirst() but for mbstring.internal_encoding
 */
static function lcfirst($string)
{
	return mb_strtolower(mb_substr($string, 0, 1)) . mb_substr($string, 1);
}

/**
 * Uppercase first character of each word similar to ucwords() but for mbstring.internal_encoding
 */
static function ucwords($string)
{
	return preg_replace_callback('/\b\w/u', function($m) { return mb_strtoupper($m[0]); }, mb_strtolower($string));
}

static function substr_replace($string, $replacement, $start, $length)
{
	return grapheme_substr($string, 0, $start) . $replacement . grapheme_substr($string, $start + $length);
}

/**
 * Extract key => value pairs from assoc array by key
 * @param $array array to filter
 * @param $keys array or comma separated list of keys to keep
 * @param $p['reorder'] reorder pairs to the order of the $keys array
 */
static function filter_keys($array, $keys, $p = array())
{
	$result = array();
	if (is_string($keys))
		$keys = explode(",", $keys);
	if ($p['reorder'])
	{
		foreach ((array)$keys as $key)
			$result[$key] = $array[$key];
	}
	else
	{
		$keep = array_flip((array)$keys);
		foreach ($array as $key => $val)
			if (isset($keep[$key]))
				$result[$key] = $val;
	}

	return $result;
}

/**
 * Construct shell command using it::shell_command, log it, execute it and return output as string.
 * {keyword} quotes and inserts value from assoc array like ET()
 * {0} .. {n} quotes and inserts positional arguments
 * {-opts} array of opts => {value,true,false,null}: it::exec('ls {-opts}', ['-opts' => ["-l" => true]]);
 * @param $cmd Format string with {keywords} a la ET()
 * @param $args varargs, contains key => val arrays for filling in cmd line. val=null expands to nothing
 * @return output of command. shell errors not detectable, consider it::system or see  /www/server/log/error_log
 */
static function exec($cmd, ...$args)
{
	$cmd = it::shell_command($cmd, ...$args);

	$before = gettimeofday(true);
	$result = EDC('noexec') ? "" : (string)shell_exec("set +o posix\n" . $cmd);

	@it::log('exec', round((gettimeofday(true) - $before)*1000) . "\t$cmd");

	return $result;
}

/**
 * Construct shell command using it::shell_command, log it, execute it and return exit code.
 * stdout/stderr is forwarded to stdout/stderror of calling script.
 * See ::exec above for usage
 * @return exit code of last command: zero for success, nonzero for failure
 */
static function system($cmd, ...$args)
{
	$cmd = it::shell_command($cmd, ...$args);

	$before = gettimeofday(true);
	if (!EDC('noexec'))
		system("set +o posix\n" . $cmd, $result);
	else
		$result = 0;

	@it::log('exec', round((gettimeofday(true) - $before)*1000) . "\t$cmd");

	return $result;
}

/**
 * Construct shell command
 * Keywords {keyword} are replace a la ET(),
 * {-opts} array of opts => {value,true,false,null}: it::exec('ls {-opts}', ['-opts' => ["-l" => true]]);
 * @param $cmd Format string with {keywords} replace a la ET()
 * @param $args (zero, one or more arrays can be passed)
 * @return output of command. shell errors not detectable, see error_log in /www/server/log
 */
static function shell_command($cmd, ...$args)
{
	$values = array();

	# Merge values into one array
	foreach ($args as $arg)
		$values = array_merge($values, (array)$arg);

	# for escapeshellarg in it::_exec_quotevalue
	$oldlocale = setlocale(LC_CTYPE, 0);
	setlocale(LC_CTYPE, 'de_CH');
	foreach (it::match('({(-?)([a-z0-9]\w*)})', $cmd, ['all' => true]) as $tags)
	{
		list($tag, $option, $key) = $tags;
		$parts = [];

		if ($option)
		{
			foreach ((array)$values["-$key"] as $key => $value)
			{
				if ($key[0] != '-')
					$key = ($key[1] ? '--' : '-') . $key;
				if ($value === true || $value === false || $value === null)
					$parts[] = $value ? $key : "";
				else foreach ((array)$value as $val)
					$parts[] = "$key " . it::_exec_quotevalue($val, "cmd=$cmd key=$key val=$val");
			}
		}
		else
		{
			foreach ((array)$values[$key] as $value)
				if ($value !== false)
					$parts[] = it::_exec_quotevalue($value, "cmd=$cmd key=$key val=$value");
		}

		$replacements[$tag] = implode(" ", $parts);
	}

	$cmd = strtr($cmd, (array)$replacements);	# Use strtr behaviour of going through source string once
	setlocale(LC_CTYPE, $oldlocale);
	return $cmd;
}

#fails with C locale!!!
static function _exec_quotevalue($value, $errmsg = "")
{
	$result = strval($value);

	if (it::match('^-', $result))
		it::fatal("leading - in value '$result': " . $errmsg);

	return preg_match('#^[-a-zA-Z0-9./_:,]+$#', $result) ? $result : escapeshellarg($result);
}


/**
 * execute a command on all cluster hosts of the current service
 * @params see it::exec()
 * @see it::exec, /opt/ultra/bin/cdist
 */
static function cexec($cmd, ...$args)
{
	return it::exec('cdist -c {cmd}', ['cmd' => it::shell_command($cmd, ...$args)]);
}


/**
 * distribute local files to all cluster hosts of the current service. no stderr if called with @
 * @params $args Variable number of options (e.g. --firsttwo), filename or array of files to distribute
 * @see it::exec, /opt/ultra/bin/cdist
 */
static function cdist(...$args)
{
	foreach ($args as $arg)
	{
		if (is_string($arg) && $arg[0] == '-')
			$opts[$arg] = true;
		else
			$files = array_merge((array)$files, (array)$arg);
	}

	# no files: return early, don't execute cdist -q ''
	if (!is_array($files) || !count($files))
		return true;

	# Check for invalid file names
	if (in_array(false, $files, true) || in_array(null, $files, true))
		return it::error("invalid file name(s)");

	return ($errs = it::exec('cdist -q {-opts} {files} 2>&1', ['-opts' => $opts, 'files' => $files])) ? it::error(['title' => $errs]) : true;
}


/**
 * Convert an image to a given size and type (ensures input is an image)
 * @param $p['in']		Input filename (mandatory)
 * @param $p['out']		Output filename (mandatory)
 * @param $p['size']		Width x height of resulting image, e.g. "160x60"
 * @param $p['type']		Output file type, e.g. "jpg"
 * @param $p['types']		Comma separated list of accepted input types, default "bmp,eps,gif,jp2,jpg,png,svg,tif"
 * @param $p['quality']		JPEG quality (0-100), default is 75
 * @param $p['keepalpha']	Don't add option --flatten to preserve alpha channel
 * @param $p['pngcrush']  Use pngcrush for sm
 * @param $p['-opts']		Custom command line options to ImageMagick convert
 * @return Success of convert as true/false
 */
static function imageconvert($p)
{
	if (!(($imagetype = @exif_imagetype($p['in'])) && ($type = image_type_to_extension($imagetype, false))))
	{
		if (get_class((object)it_xml::create(it::fopen($p['in'], "r"), array('prefix' => "_imageconvert_", 'safety' => 0))) == "_imageconvert_svg")
			$type = "svg"; # Accept SVG files if they are valid XML and root tag is svg
		else
			list(, $type) = explode(' ', strtolower(it::exec('identify 2>/dev/null {in}', $p))); # for things like eps
	}

	# emulate -auto-orient which is not supported by GM
	if ($p['-opts']['-auto-orient'])
	{
		$exif = @exif_read_data($p['in'], 'IFD0');
		switch ($exif['Orientation'])
		{
			case 2: $p['-opts']['-flop'] = true;  break;
			case 3: $p['-opts']['-rotate'] = 180; break;
			case 4: $p['-opts']['-flip'] = true;  break;
			case 6: $p['-opts']['-rotate'] = 90;  break;
			case 8: $p['-opts']['-rotate'] = 270; break;
		}

		unset($p['-opts']['-auto-orient']);
	}

	$type = strtr($type, array("jpeg" => "jpg", "tiff" => "tif", "ps" => "eps", "ept" => "eps"));
	$p += array('type' => $type, 'types' => "bmp,eps,gif,jp2,jpg,png,svg,tif", 'quality' => 75);
	$p['-opts'] = array('-thumbnail' => $p['size'], '-quality' => $p['quality']) + (array)$p['-opts'];

	if (!$p['keepalpha'])
		$p['-opts'] = array_merge(array('-flatten' => true), $p['-opts']);	# Option -flatten must be first

	$ultratimeout = file_exists("/opt/ultra/bin/ultratimeout") ? "/opt/ultra/bin/ultratimeout 30 " : "";

	if (in_array($type, explode(',', $p['types'])))	# Valid type?
		$cmdoutput = it::exec('( ' . $ultratimeout . 'gm convert -limit threads 2 +profile "*" 2>&1 {-opts} {in} {type}:{out} || echo "SHELL ERROR $?" ) | grep -Ev "( iCCP: | Invalid SOS parameters for sequential JPEG | profile matches .* but writing .* instead | inconsistent chromacities )"', $p);

	if ($p['pngcrush'] && $p['type'] == "png")
		it::exec('pngcrush.sh 2>/dev/null {out} {out}.tmp && mv {out}.tmp {out} || rm {out}.tmp', $p);

	return $cmdoutput === "";
}


/**
 * Parse command line options with Usage given as template and return assoc array.
 * @param $usage Usage parsed to determine options, example see below.
 * @param $p['optbeforearg'] Stop parsing options when first argument is encountered
 * @param $p['noexit'] Just return false in case of help or bad usage
 * @return Associative array of options, key 'args' for optional non-option args
 *
 * Usage: myprogram.php [OPTIONS] PATTERN [MOREARGS]
 *  -s, --short      Use short ouput
 *  -f, --file=FILE  Use FILE for input [foo.txt]
 *  -x EXTENSION     Ignore EXTENSION
 * Mandatory arguments from the Usage: line are returned under their (lowercased!) name.
 * Option text must be indented; if long and short option present, value is stored in long one.
 * For long opts with arguments, = is mandatory in usage. Defaults in [ ] at end of line
 * Two or more blanks must be in front of option explanation
 * Options without arguments store true or false under their key
 * Option --help or -h prints de-indented usage and exits (needs not be in usage)
 * Option --debug foo-bar will store bar in EDC('foo'). (needs not be in usage)
 * Option --verbose sets debug variable verbose for EDC('verbose', $bar). (must be in usage)
 */
static function getopt($usage, $p = array())
{
	$GLOBALS['it_stdin'] = array(
		'fd' => null,
		'args' => array(),
		'filename' => "",
		'line' => 0,
	);

	$result = array('args' => array());

	if ($indentation = it::match('^\s+', $usage))
		$usage = it::replace(array($indentation => "\n"), $usage);

	foreach(explode("\n", trim($usage)) as $usageline)
	{
		$shortoptname = $shortoptarg = $longoptname = $longoptarg = "";
		foreach (explode(',', $usageline, 2) as $optdesc)
		{
			$optdesc = trim($optdesc);
			if ($matches = (array)it::match('^--(\w[\w-]*)(=[A-Z])?', $optdesc))
				list($longoptname, $longoptarg) = $matches;
			else if ($matches = (array)it::match('^-(\w)( [A-Z])?', $optdesc))
				list($shortoptname, $shortoptarg) = $matches;
		}

		if (($default = it::match('\[(.*)\]\s*$', $usageline)) !== null)
		{
			if ($longoptarg || $shortoptarg)
				$defaults[$longoptname ? $longoptname : $shortoptname] = it::replace(array('^default:\s*' => ""), trim($default));
			else if (($longoptname || $shortoptname) && it::match('^\w+$', $default))
				it::error('defaults for boolean arguments not supported!');
		}

		if ($longoptname || $shortoptname)
		{
			if ($longoptname && ($shortoptname || $shortoptname === "0"))
			{
				if ($alias[$shortoptname])
					it::error("getopt: short option -$shortoptname is used twice: --{$alias[$shortoptname]} and --$longoptname!");
				$alias[$shortoptname] = $longoptname;
			}

			$witharg[$longoptname ? $longoptname : $shortoptname] = $longoptarg || $shortoptarg;
		}
	}
	$witharg['debug'] = true;

	$argsdesc = trim(it::replace(array("\n.*" => "", "^\S+\s+\S+\s*" => ""), trim($usage)));
	$mandatoryargs = it::match('\S+', it::replace(['\[.*?\]' => ''], $argsdesc), ['all' => true]);
	$optionalargs = it::match('\[((?!OPTIONS\]).*?)\]', $argsdesc, ['all' => true]);
	if ($mandatoryargs && !it::match("Usage:", $usage))
		it::error("Usage string must contain 'Usage:'");

	foreach (array_slice($_SERVER['argv'], 1) as $arg)
	{
		if ($eat)
			$result[array_shift($eat)] = $arg;
		else if ($arg == "--")
			$noopts = true;
		else if (!$noopts && ($matches = (array)it::match('^--(\w[\w-]*)(=.*)?', $arg)))
		{
			list($optname, $val) = $matches;
			if (!isset($witharg[$optname]) || isset($val) && !$witharg[$optname])
				$err = true;
			else if($witharg[$optname] && !$val)
				$eat[] = $optname;
			else
				$result[$optname] = $val ? substr($val, 1) : true;
		}
		else if (!$noopts && (($letters = it::match('^-(\w+)', $arg)) || $letters === "0"))
		{
			foreach (explode("\n", trim(chunk_split($letters, 1, "\n"))) as $letter)
			{
				$optname = $alias[$letter] ? $alias[$letter] : $letter;
				if ($witharg[$optname] && it::match('=', $arg))
					$result[$optname] = it::match('=(.*)', $arg);
				else if ($witharg[$optname])
					$eat[] = $optname;
				else if (!isset($witharg[$optname]))
					$err = true;
				else
					$result[$optname] = true;
			}
		}
		else if($mandatoryargs)
			$result[strtolower(array_shift($mandatoryargs))] = $seenarg = $arg;
		else
			$result['args'][] = $seenarg = $arg;

		if ($p['optbeforearg'] && $seenarg)
			$noopts = true;
	}

	if ($err || $eat || $result['h'] || $result['help'] || $mandatoryargs || (!$optionalargs && $result['args']))
	{
		if (is_resource($out = $result['h'] || $result['help'] ? STDOUT : STDERR))
			fputs($out, trim($usage) . "\n");
		return $p['noexit'] ? false : exit(1);
	}

	if ($result['debug'])
		it_debug::set($result['debug']);

	$GLOBALS['it_stdin']['args'] = $result['args'] ? $result['args'] : array("-");

	if ($result['verbose'])
		$GLOBALS['debug_verbose']++;

	return $result + (array)$defaults;
}

static function _stdin_next()
{
	if ($result = $GLOBALS['it_stdin']['args'])
	{
		$GLOBALS['it_stdin']['filename'] = $filename = array_shift($GLOBALS['it_stdin']['args']);
		$GLOBALS['it_stdin']['fd'] = $filename == "-" ? STDIN : (file_exists($filename) ? it::fopen($filename, "r") : false);
		$GLOBALS['it_stdin']['line'] = 0;
	}

	return $result;
}

/**
 * Get one line from stdin (or files given on command line) a la Perl <>.
 * Note: You need to call getopt() before using this function.
 * @return Line (including newline) or false on eof
 */
static function gets()
{
	static $initialized = 0;

	if (!$initialized++)
		it::_stdin_next();

	do {
		$result = fgets($GLOBALS['it_stdin']['fd']);
	} while (($result === false) && it::_stdin_next());

	$GLOBALS['it_stdin']['line']++;
	return $result;
}

/**
 * Like time() but overridable via debug param
 */
static function time()
{
	return ($t = EDC('time')) ? strtotime(rtrim(it::replace(array('(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)-(\d\d)(-(\d\d))?' => '$1-$2-$3 $4:$5:$7', '^(\d\d)-?(\d\d)(-?(\d\d))?$' => 'today $1:$2:$4'), $t), ":")) : time();
}

/**
 * Output formatted and localized date
 * @param format optional format (default is 2007-01-02 03:04:05).
 *        Other formats are "date", "datetime", "time".
 *        Formats can be qualified with language, e.g. "date:en"
 *        Special formats without language support are "icsdate", "icsdatetime".
 * @param stamp optional unix timestamp (default is now).
 *        If it contains nondigits, it is fed to strtotime
 */
static function date($format = "", $stamp = null)
{
	if (!isset($stamp))
		$stamp = it::time();
	else if (is_string($stamp) && !ctype_digit($stamp))
		$stamp = strtotime(it::replace(['\s(--|\+\+)\s*(\d)' => ' +\2', '\s(\+-|-\+)\s*(\d)' => ' -\2'], $stamp), it::time());

	list($name, $language) = explode(":", $format);

	if ($format && !$language)
		$language = T_lang();

	$formats = array(
		"" => "Y-m-d H:i:s",
		"date" => "d.m.Y",
		"datetime" => "d.m.Y H:i",
		"time" => "H:i",
		"date:en" => "m/d/Y",
		"datetime:en" => "m/d/Y H:i",
		"icsdate" => "Ymd",
	);

	if (!($formatstring = $formats["$name:$language"]) && !($formatstring = $formats[$name]))
	{
		if ($format == "icsdatetime")	# Special icsdate or icsdatetime format? Use UTC time format for Google Calendar to be happy
			return gmdate("Ymd", $stamp) . "T" . gmdate("His", $stamp) . "Z";
		else
			$formatstring = $format;
	}

	return date($formatstring, intval($stamp));
}

# Internal: Convert expression or funcname or function literal to callable
static function createfunc($code)
{
	if ($code instanceof Closure && (new ReflectionFunction($code))->getNumberOfRequiredParameters() == 1)
		$code = function($dummy, $v) use ($code) {return $code($v);};

	if (is_string($code) && it::match('^[\w:]+$', $code) && is_callable($code))
		$code .= '($v)';

	if (!is_callable($func = $code) && !function_exists($func = "_it_map_" . md5($code)))
		eval("function $func(\$k, \$v) { return $code; }");	# Anonymous functions/closures are slower than this (PHP 7.1)

	return $func;
}

/**
 * Iterate over an array, replacing every element by function value or expression
 * @param $code The function (gets val or key,val depending on argcount) or string expression (gets $k and $v) to apply. Returns new val
 * @param $array The array to iterate over
 * @param $p['keys'] Only modify elements with given keys (keys can be array or comma separated)
 */
static function map($code, $array, $p = null)
{
	$func = self::createfunc($code);
	$result = is_array($array) ? $array : iterator_to_array($array);
	foreach (isset($p['keys']) ? it::filter_keys($result, $p['keys']) : $result as $k => $v)
		$result[$k] = $func($k, $v);

	return (array)$result;
}

/**
 * Return only elements of $array for which $code argument evaluates to true. Preserves keys
 * @param $code The function (gets key and value as args) or string expression (gets $k and $v) to evaluate
 */
static function filter($code, $array)
{
	$func = self::createfunc($code);
	foreach (is_array($array) ? $array : iterator_to_array($array) as $k => $v)
		if ($func($k, $v))
			$result[$k] = $v;

	return (array)$result;
}


/**
 * Send a mail.
 * @param $p Header => Content (e.g To => me@example.com, Body => bodytext, Html => htmlbodytext)
 * @param $p['forcemail'] Send this mail even if we're on a twin or devel machine
 * @param $p['it_error']  Error handling params for bad email addresses
 * @return false on some errors but most cannot be caught
 */
static function mail($p)
{
	$headers = $p;
	unset($headers['forcemail'], $headers['it_error'], $headers['Body'], $headers['Html']);
	$mail = new it_mail(array_filter($headers));
	$mail->add_body($p['Body']);
	$mail->add_body($p['Html'], IT_MAIL_HTML);

	return $mail->send($p);
}

/**
 * Reads a file and returns it as string or in one of several formats. Two params: filename and flags
 * @param $filename name of file to read or - for stdin
 * @param $p['keyval'] each line of the file is one tab-separated key/value pair, return assoc array
 * @param $p['lines'] return file as array of lines without newline
 * @param $p['json'] return data json-decoded, set to "assoc" to get objects as assoc arrays
 */
static function file_get($filename, $p = array())
{
	if (($data = it::file_get_contents($filename == "-" ? "php://stdin" : $filename)) !== false)
	{
		if ($p['keyval'])
		{
			foreach (explode("\n", rtrim($data, "\n")) as $line)
			{
				$arr = explode("\t", $line, 2);
				$result[$arr[0]] = $arr[1];
			}
		}
		else if ($p['lines'])
			$result = explode("\n", rtrim($data, "\n"));
		else if ($p['json'])
			$result = it::json_decode($data, ['assoc' => $p['json'] == "assoc", 'it_error' => ['title' => "bad json in $filename"]]);
		else
			$result = $data;
	}

	return $result;
}

/**
 * Atomically write data to a file with several serialization modes
 * @param $filename name of file to write or - for stdout
 * @param $data data to write, string by default
 * @param $p['keyval'] $data must be an assoc array and is written as tab-separated lines
 * @param $p['lines'] write array of lines, appending newline
 * @param $p['json'] write array as json (prettyprints on devel)
 * @param $p['mkdir'] create parent directory (one level)
 * @param $p['cdist'] distribute saved file
 * @return falsey for write errors
 */
static function file_put($filename, $data, $p = array())
{
	$filename = it::safe_filename($filename);

	if ($p['keyval'])
		$data = implode("", it::map('"$k\t$v\n"', $data));
	else if ($p['lines'])
		$data = count((array)$data) ? implode("\n", (array)$data) ."\n" : "";
	else if ($p['json'])
		$data = it::json_encode($data);

	if ($p['mkdir'] && $filename != "-")
		@mkdir(dirname($filename));

	$tmp = $GLOBALS['ULTRAHOME'] . "/tmp";
	$tmpdir = is_dir($tmp) && is_writable($tmp) && stat($tmp)[0] == stat(dirname($filename))[0] ? $tmp : dirname($filename);
	$tmpfile = "$tmpdir/.it_put." . basename($filename) . "." . getmypid();
	if (($result = it::file_put_contents($filename == "-" ? "php://stdout" : $tmpfile, $data)) !== false && $filename != "-")
		if (($result = rename($tmpfile, $filename)) && $p['cdist'])
			$result = it::cdist($filename);

	return $result;
}

/**
 * Create an it_pipe from a file
 * @param $p if scalar or numeric array, file name(s) to read
 * @param $p['data'] array of lines without \n to feed into pipe
 */
static function cat($p = null)
{
	return new it_pipe(is_array($p) && array_key_exists('data', $p) ? $p : ['fn' => $p]);
}

/**
 * Create an it_pipe from a command or a lines array
 * @param $cmd command to execute, or if it is an array, the lines to use
 * @param $args named arguments to fill into cmd. see it::exec for usage
 */
static function pipe($cmd, $args = array())
{
	return new it_pipe(is_string($cmd) ? array('cmd' => $cmd, 'args' => $args) : array('data' => $cmd));
}

/**
 * Encodes data to json with unescaped slashes, unescaped unicode and on devel pretty printing
 * @param $p['pretty'] Force enable or disable pretty printing
 */
static function json_encode($data, $p = [])
{
	return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | ($p['pretty'] ?? it::is_devel() ? JSON_PRETTY_PRINT : 0));
}

/**
 * Decodes json data and provides warning if failed
 * $p['assoc'] Return objects as assoc arrays
 */
static function json_decode($json, $p = [])
{
	return ($data = json_decode($json, $p['assoc'])) === null && $json != 'null' ? it::error((array)$p['it_error'] + ['title' => "invalid json", 'body' => $json]) : $data;
}

/**
 * Retuns sorted array.
 * @param $mode contains combinations of a for assoc, k for key, r for reverse and n for numeric
 * @param $compare two arg compare function or single arg map function. $mode can k a "" or omitted
 */
static function sort($array, $mode = "", $compare = null)
{
	if (is_callable($mode) && !(is_string($mode) && strlen($mode) < 3)) # "a" is a func and a mode
		list($mode, $compare) = [ "", $mode ];

	if (!($arg = $compare))
		$arg = it::match('n', $mode) ? SORT_NUMERIC : 0;
	else if (($prefix = "u") && (new ReflectionFunction($compare))->getNumberOfRequiredParameters() == 1)
		$arg = function($a, $b) use ($compare) { return $compare($a) <=> $compare($b); };

	$func = $prefix . it::replace(array('n' => ""), count_chars($mode, 3)) . "sort"; # count_chars sorts flags
	$func($array, $arg);

	return $array;
}

/**
 * returns $a modulo $n as a positive number between 0 and $n - 1
 */
static function mod($a, $n)
{
	return (($a % $n) + $n) % $n;
}

static function safe_filename($filename)
{
	if (it::match("\.\./", $filename))
		it::fatal(['title' => "../ contained in '$filename', aborted"]);

	return $filename;
}

static function file_get_contents($filename, ...$args)
{
	return file_get_contents(it::safe_filename($filename), ...$args); # NOPHPLINT
}

static function file_put_contents($filename, $data, $flags = 0, $resource = null)
{
	return file_put_contents(it::safe_filename($filename), $data, $flags, $resource); # NOPHPLINT
}

static function fopen($filename, $mode, $use_include_path = false)
{
	return fopen(it::safe_filename($filename), $mode, $use_include_path); # NOPHPLINT
}

static function file($filename, $flags = 0, $context = null)
{
	return file(it::safe_filename($filename), $flags, $context); # NOPHPLINT
}

static function readfile($filename, $use_include_path = false, $context = null)
{
	return readfile(it::safe_filename($filename), $use_include_path, $context); # NOPHPLINT
}

/**
 * Convert input ($_GET, $_POST, $_REQEST, $_COOKIE and relevant $_SERVER vars, argv) to utf-8
 * and remove all params that have numeric keys from $_GET and $_REQUEST
 */
static function params2utf8()
{
	if ($GLOBALS['argv'])
		$GLOBALS['argv'] = it::any2utf8($GLOBALS['argv']);

	if ($_SERVER['argv'])
		$_SERVER['argv'] = it::any2utf8($_SERVER['argv']);

	$_GET = it::any2utf8($_GET);
	$_REQUEST = it::any2utf8($_REQUEST);
	$_POST = it::any2utf8($_POST);
	$_COOKIE = it::any2utf8($_COOKIE);

	foreach (['PHP_SELF', 'SCRIPT_NAME', 'SCRIPT_URL', 'SCRIPT_URI', 'HTTP_USER_AGENT'] as $var)
		$_SERVER[$var] = it::any2utf8($_SERVER[$var]);

	$urlfix = function($m) { return urlencode(it::any2utf8(urldecode($m[0]))); }; # NOPHPLINT
	foreach (['QUERY_STRING', 'REQUEST_URI', 'HTTP_REFERER'] as $var)
	{
		$_SERVER[$var.'_RAW'] = $_SERVER[$var];
		$_SERVER[$var] = it::any2utf8($_SERVER[$var]);

		if (strpos($_SERVER[$var], '%') !== false)
		{
			if (grapheme_strlen(urldecode($_SERVER[$var])) === null) # NOPHPLINT handle latin (double encodes correct utf8)
				$_SERVER[$var] = preg_replace_callback('/%[89A-F][A-Z0-9]/i', $urlfix, $_SERVER[$var]);
			while (preg_match('/%C3%8[23]%C2%[89ab][0-9a-f]/i', $_SERVER[$var]) && $iterations++ < 3) # handle doubly encoded utf8, UTF8SAFE
				$_SERVER[$var] = preg_replace_callback('/%C3%8[23]%C2%[89ab][0-9a-f]/i', $urlfix, $_SERVER[$var]);
		}
	}

	$_SERVER['HTTP_REFERER'] = it::replace(['#[^#]*$' => ""], $_SERVER['HTTP_REFERER']); # safari sometimes puts anchor in referer

	foreach (array_filter(array_keys($_REQUEST), 'is_int') as $key) # remove numeric keys because they confuse U()
		unset($_REQUEST[$key], $_GET[$key]);
}

/**
 * Compatibility function to map new setcookie() parameters to pre-7.3 PHP version
 */
static function setcookie($name, $value, $p = [])
{
	return version_compare(PHP_VERSION, '7.3.0', '<')
		? @setcookie($name, $value, $p['expires'], $p['path'], $p['domain'], $p['secure'], $p['httponly'])
		: @setcookie($name, $value, $p);
}

static function urldecode($str)
{
	return it::any2utf8(urldecode($str)); # NOPHPLINT
}

static function parse_str($str) # note different usage
{
	parse_str($str, $request); # NOPHPLINT

	return it::any2utf8($request);
}

static function request_body()
{
	return it::any2utf8(it::file_get_contents('php://input'));
}

static function utf8_decode($utf8)
{
	return UConverter::transcode($utf8, 'ISO-8859-1', 'UTF8', ['to_subst' => '?']);
}

static function utf8_encode($latin1)
{
	return UConverter::transcode($latin1, 'UTF8', 'ISO-8859-1');
}

}