<?php
/*
**	Copyright (C) 1995-2007 by the ITools Authors.
**	This file is part of ITools - the Internet Tools Library
**
**	ITools is free software; you can redistribute it and/or modify
**	it under the terms of the GNU General Public License as published by
**	the Free Software Foundation; either version 3 of the License, or
**	(at your option) any later version.
**
**	ITools is distributed in the hope that it will be useful,
**	but WITHOUT ANY WARRANTY; without even the implied warranty of
**	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**	GNU General Public License for more details.
**
**	You should have received a copy of the GNU General Public License
**	along with this program.  If not, see <http://www.gnu.org/licenses/>.
**
**	it.class - static functions
*/

class it
{
/**
 * Create config class with static members initialized (e.g. $home).
 * NOTE: PHP5 ONLY
 * @param $p Static members to be generated in newly created class
 *      service: Class name of created class (default: from caller path)
 *      home: Home directory of application (default: from caller path)
 *      site: Domain name of application (default: from caller path)
 *      db: Database of application (default: from caller path)
 */
static function createconfig($p = array())
{
	$stack = debug_backtrace();
	$filename = $stack[0]['file'];
	preg_match('!/www/((\w+)[^/]+)!', $filename, $parts);

	$p += array(
		'home' => $parts[0],
		'site' => $parts[1],
		'db' => strtr($parts[1], ".-", "__"),
		'service' => $parts[2],
	);

	if (class_exists($p['service'] . "_tools"))
		$extends = "extends {$p['service']}_tools ";

	$code = array("class {$p['service']} $extends{");

	foreach ($p as $name => $value)
		$code[] = "static \$$name = " . var_export($value, true) . ";";

	$code[] = "}";
	eval(join("\n", $code));
}


/**
 * Clone an object and return copy, works for all PHP versions
 */
static function &cloneobj(&$object)
{
	$result = (is_object($object) && version_compare(zend_version(), 2, '>=')) ? clone($object) : $object;

	return $result;	# PHP internals need a tmp var to return by ref
}


/**
 * Append all arguments to a logfile (tab separated). Date will be added to filename and line
 * @param $name Name of logfile. Will be in log/ of service unless it starts with /
 * @param $args... Varargs to log, will be tab separated.
 */
static function log($name /* ... */)
{
	$args = func_get_args();
	$line = date("Y-m-d H:i:s") . "\t" . implode("\t", array_slice($args, 1));
	self::log_line($name, $line);
	return $line;
}

/**
 * Append line to a logfile. Date will be added to filename
 * @param $name Name of logfile. Will be in log/ of service unless it starts with /
 * @param $line Line to log.
 */
static function log_line($name, $line)
{
	$basefn = substr($name, 0, 1) == "/" ? $name : $GLOBALS['ULTRAHOME'] . "/log/$name";
	$fullfn = $basefn . "-" . date('Ymd');

	if (substr($fullfn, 0, 1) == "/")
	{
		if (!file_exists($fullfn))
		{
			$tmp = getmypid();
			@touch("$fullfn.$tmp");
			@chgrp("$fullfn.$tmp", "www");
			@chmod("$fullfn.$tmp", 0664);
			@rename("$fullfn.$tmp", $fullfn);
			@unlink($basefn);
			if (@readlink($basefn) < basename($fullfn)) # may be older due to $GLOBALS['debug_time']
				@symlink($fullfn, $basefn);
		}

		file_put_contents($fullfn, $line . "\n", FILE_APPEND);
	}	
}


/**
 * Store timings for appending to log/timer_log-* in auto_append.php
 */
static function timerlog($label = '')
{
	if ($GLOBALS['debug_timerlog'])
		$GLOBALS['ULTRATIMERLOG'] .= sprintf(" %s:%d", $label, (microtime(true) - $GLOBALS['ULTRATIME'])*1000);
}


/**
 * If display_errors is on or stdout is a tty, shows error in page or on stdout respectively
 * If display_errors is off, mails (with rate limiting) diagnostics to .diffnotice addresses or file owner or SERVER_ADMIN
 * @param $p either error title or assoc array of params, see below
 * @param $p['title'] error title, one line. also accepted in $p[0] (with priority). false means ignore error
 * @param $p['body'] error body: multiline string or any data type (will be dumped)
 * @param $p['to'] comma separated recipient list
 * @param $p['id'] id of error, used with timewindow, defaults to file and line of caller
 * @param $p['graceperiod'] in seconds. deprecated, see $p['timewindow']
 * @param $p['timewindow'] in secs. "5-35" means for an notice to be sent, a second error must occur 5 to 35 seconds after first err
 * @param $p['blockmailid'] block mail for $p['blockmail'] seconds with same id. Default: $p['to']
 * @param $p['blockmail'] number of seconds to block mails after having sent a mail [3600]
 * @param $p['backtraceskip'] number of stack levels to drop
 * @param $p['skipfiles'] files to skip in backtrace
 * @param $p['okstate'] give current ok label and ok state, e.g. telresult=1 for working. see failcount
 * @param $p['failcount'] give number of consecutive okstate failures needed for creating an error [2]
 * @param $p['omitdebuginfo'] value 1 omits long stack and var dumps, value 2 also minimal infos
 * @param $p['fatal'] exit after displaying error
 * @return always null (so users can return it::error() in error cases)
 *
 * TIMEWINDOW 5-35 (x = error occurs, no mail; m = error occurs, mail sent)
 *   time        ---|5 secs|30 secs      |------|5 secs|30 secs
 *                                                          |5 secs|30 secs
 *                  x  x                        x   x       m   x     m   x
 * BLOCKMAIL
 *   time        ---|blockmail|--|blockmail|---->
 *   blockmailid    m  x     x   m   x
 *
 * GRACPERIOD/TIMEWINDOW (DEPRECATED, graceperiod is first argument of 5-35, timewindow is difference between 5 and 35)
 *   time        ---|graceperiod|timewindow|
 *                                   |graceperiod|timewindow|--.
 *                                                             |graceperiod|timewindow|------>
 *   id             x  x     x       m    x   x                x     x
 */
static function error($p = array())
{
	$p = $origp = (array)$p;
	$p['title'] = $p[0] ?: $p['title']; # handle 'it_error' => "oops" that was cast to array on the way

	if (!error_reporting() || $p[0] === false || $p['title'] === false) # called with @ or suppressed
		return null;

	if ($_SERVER['REMOTE_ADDR'])
		$url = "http://" . $_SERVER['HTTP_HOST'] . $_SERVER['REQUEST_URI']; # we ignore https for easier debugging
	else
		$url = $_SERVER['SCRIPT_NAME'] . " " . join(" ", array_slice($GLOBALS['argv'], 1)) . " (pwd " . $_SERVER['PWD'] . ")";

	$gods = strtr(trim(@file_get_contents($GLOBALS['ULTRAHOME'] . "/.diffnotice")), array("\n"=>', '));
	if (!$p['to'])
		unset($p['to']); # allow defaults to kick in
	$p += array(
		'title' => "it::error",
		'to' => $gods ? $gods : (get_current_user() ? get_current_user() : $_SERVER['SERVER_ADMIN']),
		'graceperiod' => 60, 
		'timewindow' => 25*3600,
		'backtraceskip' => 1,
		'blockmail' => $p['okstate'] ? 60 : 3600,
		'omitdebuginfo' => false,
		'failcount' => 2,
		'id' => $p['timewindow'] ? "it_error_id_" . it_debug::backtrace(['skipfiles' => "it.class", 'levels' => 1]) : 0,
	);
	if (it::match('-', $p['timewindow']) && ($parts = explode("-", $p['timewindow'])))
		list($p['graceperiod'], $p['timewindow']) = [$parts[0], $parts[1] - $parts[0]];
	$p += array('blockmailid' => $GLOBALS['ULTRASITE'] . "." . md5($p['to']));
	$trace = it_debug::backtrace(array('skiplevels' => $p['backtraceskip'], 'skipfiles' => $p['skipfiles'])); # moved in here for performance in mass error case

	@mkdir("/tmp/alertdata");
	@chmod("/tmp/alertdata", 0777);		

	if ($p['okstate'])
	{
		list($okfn, $okstatus) = explode("=", "/tmp/alertdata/okstate_" . $p['okstate']);
		$failcount = $okstatus ? 0 : (int)@file_get_contents($okfn) + 1;
		file_put_contents($okfn, "$failcount\n");
		if ($failcount != $p['failcount'] && $failcount != $p['failcount'] * 2)
			return;
	}

	$toscreen = getenv('NOERRORMAILS') || ini_get('display_errors') || (defined("STDOUT") && posix_isatty(STDOUT)) || EDC('astwin') || EDC('asdevel');

	if ($toscreen && !it::is_live())
		$GLOBALS['debug_noredir'] = 1;

	if (!$toscreen) # this error can only be sent by mail: find out if we want to suppress it
	{
		if (!$p['id'])
			$sendmail = true;
		else
		{
			$errstampfn = "/tmp/alertdata/errstamp_" . urlencode($p['id']);
			$errstamp = @filemtime($errstampfn);
			$errstampage = time() - $errstamp;
			$sendmail = $errstampage >= $p['graceperiod'] && $errstampage <= $p['graceperiod'] + $p['timewindow'];
			if ($errstampage > $p['graceperiod'])
			{
				@unlink($errstampfn);
				@touch($errstampfn);
			}
		}

		if ($sendmail)
		{
			$lastsentfn = "/tmp/alertdata/it_error_mailsent_" . urlencode($p['blockmailid']);
			clearstatcache();
			$sendmail = time() - max(@filemtime($lastsentfn), it_cache::get($lastsentfn, ['distributed' => true])) > $p['blockmail'];
			if ($sendmail)
			{
				@unlink($lastsentfn);
				@touch($lastsentfn);
				if (it::is_live())
					it_cache::put($lastsentfn, time(), ['distributed' => true, 'ttl' => 7*86400, 'safety' => 0]);
			}
		}
	}

	if ($p['fatal'] && $_SERVER['REMOTE_ADDR'] && !headers_sent())
		http_response_code(500);

	if ($toscreen || $sendmail)
	{
		$p['body'] = is_string($p['body']) || !array_key_exists('body', $p) ? $p['body'] : var_export($p['body'], true);
		if (strlen($p['body']) > 500000)
		{
			file_put_contents($datafn = "/tmp/alertdata/error-" . substr(md5($p['body']), 0, 2), $p['body']);
			$p['body'] = "Body: " . getenv('HOSTNAME') . ":$datafn";
		}

		$body =
			($p['omitdebuginfo'] >= 2 ? "" : ($url && !$toscreen? "{$p['title']}\n\nUrl: $url\n\n" : "") .
			($trace ? ($sendmail ? "" : "  ") . "Trace: $trace\n\n" : "")) .
			(!$sendmail || $p['omitdebuginfo'] >= 2 ? "" : "Host: " . getenv('HOSTNAME') . "\n\n") .
			(!$sendmail || $p['omitdebuginfo'] >= 2 ? "" : "Time: " . date("Y-m-d H:i:s") . (($t = time() - $_SERVER['REQUEST_TIME']) ? " (started {$t}s before)" : "") . "\n\n") . # no it::date() due to time- debug param
			($p['body'] ? ($p['omitdebuginfo'] ? "" : "Body:\n") . trim($p['body'])."\n\n" : "");

		if ($sendmail) # we're mailing: send maximum info
		{
			$p['title'] = it::replace(['alert:|server:|^: "' => "", '(pw|passw|password|secret)(=)[^&\s]*' => '$1$2*****'], it::match('[^.]*', $GLOBALS['ULTRASITE']) . ": " . $p['title']) . " (via " . getenv('HOSTNAME') . ")";

			if (!$p['omitdebuginfo'])
			{
				foreach (($trace ? debug_backtrace() : array()) as $level => $stackframe)
					if ($level >= $p['backtraceskip'] && $tracesize < 100000 && ($tracesize += strlen(print_r($stackframe, true))) < 100000) # save mem
						$stackframes[] = $stackframe;

				$body .= $p['id']     ? "Filter: timewindow=" . $p['graceperiod'] . "-" . ($p['graceperiod'] + $p['timewindow']) . "  (previous err: " . it::date('', $errstamp) . ")\n\n" : "";
				$body .= $origp['blockmail'] ? "Block-resend: " . $origp['blockmail'] . " seconds\n\n" : "";
				$body .= !$trace ? "" : "Overview: ssh " . getenv('HOSTNAME') . " 'cat /tmp/alertdata/alertlog | grep \"$trace\"'\n\n";
				$body .= ($trace and $t = it::exec('grep -h {0} `ls /tmp/alertdata/alertlog-*|tail -3` /dev/null 2>/dev/null | grep ^2 | cut -d : -f 1-2 | sort | uniq -c | tail -10', $trace)) ? "Histogram: (today's last 10 affected minutes)\n$t\n" : "";
				$body .= $_GET        ? "\$_GET: "    . var_export($_GET, true) . "\n\n" : "";
				$body .= $_POST       ? "\$_POST: "   . var_export($_POST, true) . "\n\n" : "";
				$body .= $_COOKIE     ? "\$_COOKIE: " . var_export($_COOKIE, true) . "\n\n" : "";
				$body .= $_SERVER['REMOTE_ADDR'] ? "" : "Pstree:\n" . it::exec("pstree -als " . getmypid() . " | head -n -3") . "\n";
				$body .= $_SERVER     ? "\$_SERVER: " . var_export($_SERVER, true) . "\n\n" : "";
				$body .= $_FILES      ? "\$_FILES:  " . var_export($_FILES, true) . "\n\n" : "";
				$body .= "Processes:\n" . it::exec('ps auxf | egrep -v "rotatelogs|getbanner|logaction|httpd|systemd|sd-pam"|egrep "^www|^cron"') . "\n";
				$body .= $stackframes ? "Stack: "     . print_r($stackframes, true) . "\n\n" : "";
				$body  = it::replace(array('(pw|passw|password|secret)(\' => |\] => |=)[^&\s]*' => '$1$2********'), $body, array('utf8' => false));
			}

			it::mail(array('To' => $p['to'], 'Subject' => ($p['fatal'] ? "Fatal: " : "Error: ") . substr($p['title'], 0, 160), 'Body' => $body, 'Cc' => $GLOBALS['it_defaultconfig']['error_cc'], 'forcemail' => !it::is_devel()));
			$p['title'] = "Mail: " . $p['title'];
		}
		else if ($_SERVER['REMOTE_ADDR']) # toscreen mode: web
			echo "<pre style='z-index:10000; position:relative; background:white'>" . htmlspecialchars($p['title'] . "\n" . rtrim($body), ENT_COMPAT, "iso-8859-1") . "</pre>"; # works with iso-8859-1 or utf-8, UTF8SAFE
		else  # toscreen mode: shell (outputs to stderr)
			error_log(substr($p['title'], 0, 100000) . " in " . ($trace ? $trace : " {$p['file']}:{$p['line']}") . " Url: $url " . (EDC('verboseerrors') ? D($p['body']) : ""));
	}

	if ($_SERVER['REMOTE_ADDR']) # additional entry in log/error_log
		error_log("it::error: " . $p['title'] . " Url: $url");

	it::log("/tmp/alertdata/alertlog", $p['title'] . " in " . ($trace ? $trace : "{$p['file']}:{$p['line']}") . " Url: $url" . ($p['body'] ? "\n" . substr(D($p['body']), 0, 5000) . "\n" : ""));

	if ($p['fatal'])
		exit(99);

	return null;
}


/**
 * Same as it::error(), plus exit
 * @see error()
 */
static function fatal($p)
{
	it::error(['fatal' => true, 'backtraceskip' => 2] + (array)$p);
}


/**
 * Print message to stderr and exit with error code
 */
static function bail($message = "Bailed.\n")
{
	fputs(STDERR, $message);
	exit(1);
}


/**
 * Check wether I am on a live server for this service. Honors 'aslive' and 'asdevel' debug vars
 * @return true if servertype is 'live'
 */
static function is_live()
{
	return (preg_match("/^live/", $GLOBALS['ULTRASERVERTYPE']) || !$GLOBALS['ULTRASERVERTYPE'] || EDC('aslive')) && !EDC('asdevel');
}


/**
 * Check wether I am on a development server for this service. Honors 'aslive' and 'asdevel' debug vars
 * @return true if servertype is 'devel'
 */
static function is_devel()
{
	return (preg_match("/^devel/", $GLOBALS['ULTRASERVERTYPE']) || EDC('asdevel')) && !EDC('aslive');
}


/**
 * Convert a string to ASCII-only chars, map/strip ISO-8859-1 accents
 * @param $text Text to convert
 * @return mapped/stripped version of $text contains only chars [0..127]
 */
static function toascii($text)
{
	return strtr(strtr($text,
		utf8_decode(	'ÇéâàåçêëèïîìÅÉôòûùÿøØáíóúñÑÁÂÀãÃÊËÈÍÎÏÓÔõÕÚÛÙýÝ'),
				'CeaaaceeeiiiAEoouuyooaiounNAAAaAEEEIIIOOoOUUUyY'),
		array("\xe4" => 'ae', "\xf6" => 'oe', "\xfc" => 'ue', "\xc4" => 'Ae', "\xd6" => 'Oe', "\xdc" => 'Ue', "\xe6" => 'ae', "\xc6" => 'Ae', "\xdf" => 'ss'));
}


/**
 * Check whether an IP adress lies within a given range. Supports IPv4 and IPv6
 * @param $ip IP address (192.168.42.123)
 * @param $cidrs IP range in CIDR notation (192.168.42.64/26) or array of ranges
 * @return true if $ip is within $cidr
 */
function cidr_match($ip, $cidrs)
{
	foreach ((array)$cidrs as $cidr)
	{
		list($subnet, $mask) = explode('/', $cidr);
		$ip_bin = inet_pton($ip);
		$subnet_bin = inet_pton($subnet);
		$valid_bytes = $mask ? $mask >> 3 : 42;
		$bitmask = 256 - (1 << (8 - ($mask & 7)));
		$lastbyte_matched = $bitmask ? (ord($ip_bin{$valid_bytes}) & $bitmask) == (ord($subnet_bin{$valid_bytes}) & $bitmask) : true;

		if (substr($ip_bin, 0, $valid_bytes) == substr($subnet_bin, 0, $valid_bytes) && $lastbyte_matched)
			return true;
	}

	return false;
}


/**
 * Convert regex for preg (adds and escapes delimiter, adds modifiers)
 * @param $pattern Regex to convert
 * @param $p['casesensitive'] Regex is case sensitive (omit modifier i)
 * @param $p['multiline'] add modifier m: ^ and $ match \n
 * @param $p['singleline'] add modifier s: . matches \n
 * @param $p['utf8'] add modifier u. This is the default if default_charset is utf-8, override with $p['utf8'] = false
 * @param $p['extended'] add modifier x (non signifcant whitespace)
 * @return converted regex to use with preg
 */
static function convertregex($pattern, $p = null)
{
	return '/' . strtr($pattern, array('/' => '\/')) . '/' .
		(!$p['casesensitive'] ? 'i' : '') .
		($p['multiline'] ? 'm' : '') .
		($p['singleline'] ? 's' : '') .
		($p['extended'] ? 'x' : '') .
		((!isset($p['utf8']) && ini_get('default_charset') == 'utf-8' || $p['utf8']) ? 'u' : '');
}


/**
 * Try to match string against regex. Case insensitive by default.
 * @param $pattern Regex to match against
 * @param $string String to match
 * @param $p['offset_capture'] Set flag preg_offset_capture (returns offsets with the matches).
 * @param $p['all'] Return every match as array instead of first match.
 * @param $p contains pattern modifiers, @see convertregex()
 * @return Matched string or false 
 */
static function match($pattern, $string, $p = null)
{
	if (!preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p)
		$r = preg_match('!' . $pattern . '!i' . (ini_get('default_charset') == 'utf-8' ? 'u' : ''), $string, $m); # fast path for simple patterns
	else
	{
		$flags = $p['offset_capture'] ? PREG_OFFSET_CAPTURE : 0;

		if ($p['all'])
			$r = preg_match_all(it::convertregex($pattern, $p), $string, $m, $flags | PREG_PATTERN_ORDER, $p['offset']);
		else
			$r = preg_match(it::convertregex($pattern, $p), $string, $m, $flags, $p['offset']);
	}

	if (!$r)	# no match
	{
		if (preg_last_error() == PREG_BACKTRACK_LIMIT_ERROR)
			it::error("Exceeded pcre.backtrack_limit of " . ini_get('pcre.backtrack_limit') . " bytes");
		else if (preg_last_error() == PREG_BAD_UTF8_ERROR)
			it::error("Invalid utf-8 in it::match haystack: " . substr($string, 0, 500)); # UTF8SAFE

		$result = $p['all'] ? array() : null;
	}
	else if (count($m) == 1)	# no capture
		$result = $m[0];
	else if (count($m) == 2)	# one capture
		$result = $m[1];
	else if ($p['all'] && !$p['pattern_order'])	# captures, reorder pattern_order to set_order but without first element
		$result = call_user_func_array('array_map', array_merge(array(null), array_slice($m, 1)));
	else	# captures, don't return first element (matched string)
		$result = array_slice($m, 1);

	return $result;
}


/**
 * Replace parts of a string matched by a pattern with according replacement string. See convertregex for named parameters.
 * @param $replacementes Array with patterns as keys and replacement strings as values
 * @param $string String to change
 * @param $p['limit'] limit number of replacements (default: all)
 * @return New string
 * @see convertregex for more options in $p
 */
static function replace($replacements, $string, $p = array())
{
	$encoding = ini_get('default_charset') == 'utf-8' ? 'u' : '';
	foreach ($replacements as $pattern => $dummy)
		$patterns[] = !preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p ? "!$pattern!i$encoding" : it::convertregex($pattern, $p);

	$result = preg_replace($patterns, $replacements, $string, isset($p['limit']) ? $p['limit'] : -1);

	 if ($result === null && preg_last_error() == PREG_BAD_UTF8_ERROR)
		it::error("Invalid utf-8 in it::replace haystack: " . substr($string, 0, 500)); # UTF8SAFE

	return $result;
}

/**
 * Returns only the array elements matching the given regex
 * @param $pattern     Regex to match against
 * @param $array       array to grep
 * @param $p           parameters for it::match or 'invert' => true to return non-matching lines
 * @return New array
 */
static function grep($pattern, $array, $p = array())
{
	if (!preg_match('/\\\\[wb]|[!\x80-\xff]|\[\[:/i', $pattern) && !$p)
		$result = preg_grep('!' . $pattern . '!i' . (ini_get('default_charset') == 'utf-8' ? 'u' : ''), $array, $p['invert'] ? PREG_GREP_INVERT : 0); # fast path
	else
		$result = preg_grep(it::convertregex($pattern, $p), $array, $p['invert'] ? PREG_GREP_INVERT : 0);

	return $result;
}


/**
 * Returns string split by pattern (like preg_split but utf8 and case insensitive)
 * @param $pattern Regex to match against, no delimiters
 * @param $subject String to split
 * @param $p['limit'], $p['no_empty'], $p['delim_capture'], $p['offset_capture'] See preg_split()
 */
static function split($pattern, $subject, $p = array())
{
	$flags = ($p['no_empty'] ? PREG_SPLIT_NO_EMPTY : 0) | ($p['delim_capture'] ? PREG_SPLIT_DELIM_CAPTURE : 0) | ($p['offset_capture'] ? PREG_SPLIT_OFFSET_CAPTURE : 0);

	return preg_split(it::convertregex($pattern, $p), $subject, $p['limit'] ?: -1, $flags);
}

/**
 * Convert string or array to utf8 if it was not already utf-8 before. Also handles double encoding
 * @param $value String or array to convert
 * @param $errprefix Error message to output if anything needed to be done
 * @return Same string in utf-8 encoding
 */
static function any2utf8($value, $errprefix = "")
{
	if (is_array($value))
	{
		foreach ($value as $idx => $v)
			$newarr[self::any2utf8($idx)] = is_string($v) || is_array($v) ? self::any2utf8($v, $errprefix) : $v;
		$value = (array)$newarr;
	}
	else if (is_string($value))
	{
		if (grapheme_strlen($value) === null)
			list($value, $error) = array(utf8_encode($value), utf8_encode("incorrect utf8-encoding. input=$value"));
		if (preg_match('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', $value))
			list($value, $error) = array(it::any2utf8(preg_replace_callback('/\xc3[\x82\x83]\xc2[\x82\x83\xbc\xa9\xa4\xb6\xa8\xa2\xa0\xb4\xaa\xa7\x84\xab\xae\x9c\xaf\x96\xb2\xbb\xb9\x9f]/', function($m) {return utf8_decode($m[0]);}, $value)), $errprefix ? "$errprefix: double utf8-encoding. input=$value" : "");
		if (preg_match('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', $value))
			list($value, $error) = array(preg_replace('/\xef\xb7[\x90-\xaf]|\xef\xbf[\xbe\xbf]/', " ", $value), "forbidden utf-8 character. input=$value");
		$value = preg_replace('/\xc2\xad/', '', $value);	# Kill invisible soft hyphens
		if (preg_match('/\xcc\x88/', $value))	# Normalize combining diaeresis to umlaut
			$value = strtr($value, [ "a\xcc\x88" => 'ä', "A\xcc\x88" => 'Ä', "e\xcc\x88" => 'ë', "E\xcc\x88" => 'Ë', "i\xcc\x88" => 'ï', "I\xcc\x88" => 'Ï', "o\xcc\x88" => 'ö', "O\xcc\x88" => 'Ö', "u\xcc\x88" => 'ü', "U\xcc\x88" => 'Ü' ]);
		if ($error && $errprefix)
			it::error(array('title' => "$errprefix: " . trim($error)));
	}

	return $value;
}

/**
 * Uppercase first character similar to ucfirst() but for mbstring.internal_encoding
 */
static function ucfirst($string)
{
	return mb_strtoupper(mb_substr($string, 0, 1)) . mb_substr($string, 1);
}

/**
 * Lowercase first character similar to lcfirst() but for mbstring.internal_encoding
 */
static function lcfirst($string)
{
	return mb_strtolower(mb_substr($string, 0, 1)) . mb_substr($string, 1);
}

/**
 * Uppercase first character of each word similar to ucwords() but for mbstring.internal_encoding
 */
static function ucwords($string)
{
	return preg_replace_callback('/\b\w/u', function($m) { return mb_strtoupper($m[0]); }, mb_strtolower($string));
}

static function substr_replace($string, $replacement, $start, $length)
{
	return grapheme_substr($string, 0, $start) . $replacement . grapheme_substr($string, $start + $length);
} 

/**
 * Extract key => value pairs from assoc array by key
 * @param $array array to filter
 * @param $keys array or comma separated list of keys to keep
 * @param $p['reorder'] reorder pairs to the order of the $keys array
 */
static function filter_keys($array, $keys, $p = array())
{
	$result = array();
	if (is_string($keys))
		$keys = explode(",", $keys);
	if ($p['reorder'])
	{
		foreach ((array)$keys as $key)
			$result[$key] = $array[$key];
	}
	else
	{
		$keep = array_flip((array)$keys);
		foreach ($array as $key => $val)
			if (isset($keep[$key]))
				$result[$key] = $val;
	}

	return $result;
}

/**
 * Construct shell command using it::shell_command, log it, execute it and return output as string.
 * {keyword} quotes and inserts value from assoc array like ET()
 * {0} .. {n} quotes and inserts positional arguments
 * {-opts} array of opts => {value,true,false,null}: it::exec('ls {-opts}', ['-opts' => ["-l" => true]]);
 * @param $cmd Format string with {keywords} a la ET()
 * @param $values (zero, one or more arrays can be passed)
 * @return output of command. shell errors not detectable, see error_log in /www/server/logs
 */
static function exec(/* $cmd, $values1 = array(), ... */)
{
	$args = func_get_args();
	$cmd = call_user_func_array('it::shell_command', $args);

	$before = microtime(true);
	$result = EDC('noexec') ? "" : (string)shell_exec($cmd);

	@it::log('exec', round((microtime(true) - $before)*1000) . "\t$cmd");

	return $result;
}

/**
 * Construct shell command using it::shell_command, log it, execute it and return exit code.
 * stdout/stderr is forwarded to stdout/stderror of calling script.
 * See ::exec above for usage
 * @return exit code of last command: zero for success, nonzero for failure
 */
static function system(/* $cmd, $values1 = array(), ... */)
{
	$args = func_get_args();
	$cmd = call_user_func_array('it::shell_command', $args);

	$before = microtime(true);
	if (!EDC('noexec'))
		system($cmd, $result);
	else
		$result = 0;

	@it::log('exec', round((microtime(true) - $before)*1000) . "\t$cmd");

	return $result;
}

/**
 * Construct shell command
 * Keywords {keyword} are replace a la ET(),
 * {-opts} array of opts => {value,true,false,null}: it::exec('ls {-opts}', ['-opts' => ["-l" => true]]);
 * @param $cmd Format string with {keywords} replace a la ET()
 * @param $values (zero, one or more arrays can be passed)
 * @return output of command. shell errors not detectable, see error_log in /www/server/logs
 */
static function shell_command(/* $cmd, $values1 = array(), ... */)
{
	$args = func_get_args();
	$cmd = array_shift($args);
	$values = array();

	# Merge values into one array
	foreach ($args as $arg)
		$values = array_merge($values, (array)$arg);

	# for escapeshellarg in it::_exec_quotevalue
	$oldlocale = setlocale(LC_CTYPE, 0);
	setlocale(LC_CTYPE, 'de_CH');
	foreach (it::match('({(-?)([a-z0-9]\w*)})', $cmd, array('all' => true)) as $tags)
	{
		list($tag, $option, $key) = $tags;
		$parts = array();

		if ($option)
		{
			foreach ((array)$values["-$key"] as $key => $value)
			{
				if ($key[0] != '-')
					$key = ($key[1] ? '--' : '-') . $key;
				if ($value === true || $value === false || $value === null)
					$parts[] = $value ? $key : "";
				else foreach ((array)$value as $val)
					$parts[] = "$key " . it::_exec_quotevalue($val, "cmd=$cmd key=$key val=$val");
			}
		}
		else
		{
			foreach ((array)$values[$key] as $value)
				$parts[] = it::_exec_quotevalue($value, "cmd=$cmd key=$key val=$value");
		}

		$replacements[$tag] = join(" ", $parts);
	}

	$cmd = strtr($cmd, (array)$replacements);	# Use strtr behaviour of going through source string once
	setlocale(LC_CTYPE, $oldlocale);
	return $cmd;
}

#fails with C locale!!!
static function _exec_quotevalue($value, $errmsg = "")
{
	$result = strval($value);

	if (it::match('^-', $result))
		it::fatal("leading - in value: " . $errmsg);

	return preg_match('#^[-a-zA-Z0-9./_:,]+$#', $result) ? $result : escapeshellarg($result);
}


/**
 * execute a command on all cluster hosts of the current service
 * @params see it::exec()
 * @see it::exec, /opt/ultra/bin/cdist
 */
static function cexec(/* $cmd, $values1 = array(), ... */)
{
	return it::exec('cdist -c {CMD}', array('CMD' => call_user_func_array('it::shell_command', func_get_args())));
}


/**
 * distribute local files to all cluster hosts of the current service. no stderr if called with @
 * @params $files filename or array of files to distribute
 * @see it::exec, /opt/ultra/bin/cdist
 */
static function cdist(/* $file1, ... */)
{
	foreach (func_get_args() as $arg)
		$files = array_merge((array)$files, (array)$arg);

	return ($errs = it::exec('cdist -q {FILES} 2>&1', ['FILES' => $files])) ? it::error(['title' => $errs]) : true;
}


/**
 * Convert an image to a given size and type (ensures input is an image)
 * @param $p['in']		Input filename (mandatory)
 * @param $p['out']		Output filename (mandatory)
 * @param $p['size']		Width x height of resulting image, e.g. "160x60"
 * @param $p['type']		Output file type, e.g. "jpg"
 * @param $p['types']		Comma separated list of accepted input types, default "bmp,eps,gif,jp2,jpg,png,svg,tif"
 * @param $p['quality']		JPEG quality (0-100), default is 75
 * @param $p['keepalpha']	Don't add option --flatten to preserve alpha channel
 * @param $p['pngcrush']  Use pngcrush for sm
 * @param $p['-opts']		Custom command line options to ImageMagick convert
 * @return Success of convert as true/false
 */
static function imageconvert($p)
{
	if (!(($imagetype = @exif_imagetype($p['in'])) && ($type = image_type_to_extension($imagetype, false))))
	{
		if (@get_class(it_xml::create(fopen($p['in'], "r"), array('prefix' => "_imageconvert_", 'safety' => 0))) == "_imageconvert_svg")
			$type = "svg"; # Accept SVG files if they are valid XML and root tag is svg
		else
			list(, $type) = explode(' ', strtolower(it::exec('identify 2>/dev/null {in}', $p))); # for things like eps
	}

	# emulate -auto-orient which is not supported by GM
	if ($p['-opts']['-auto-orient'])
	{
		$exif = @exif_read_data($p['in'], 'IFD0');
		switch ($exif['Orientation'])
		{
			case 2: $p['-opts']['-flop'] = true;  break;
			case 3: $p['-opts']['-rotate'] = 180; break;
			case 4: $p['-opts']['-flip'] = true;  break;
			case 6: $p['-opts']['-rotate'] = 90;  break;
			case 8: $p['-opts']['-rotate'] = 270; break;
		}

		unset($p['-opts']['-auto-orient']);
	}

	$type = strtr($type, array("jpeg" => "jpg", "tiff" => "tif", "ps" => "eps", "ept" => "eps"));
	$p += array('type' => $type, 'types' => "bmp,eps,gif,jp2,jpg,png,svg,tif", 'quality' => 75);
	$p['-opts'] = array('-thumbnail' => $p['size'], '-quality' => $p['quality']) + (array)$p['-opts'];

	if (!$p['keepalpha'])
		$p['-opts'] = array_merge(array('-flatten' => true), $p['-opts']);	# Option -flatten must be first

	$ultratimeout = file_exists("/opt/ultra/bin/ultratimeout") ? "/opt/ultra/bin/ultratimeout 30 " : "";

	if (in_array($type, explode(',', $p['types'])))	# Valid type?
		$cmdoutput = it::exec('( ' . $ultratimeout . 'gm convert 2>&1 {-opts} {in} {type}:{out} || echo "SHELL ERROR $?" ) | grep -v " iCCP: "', $p);

	if ($p['pngcrush'] && $p['type'] == "png")
		it::exec('pngcrush.sh 2>/dev/null {out} {out}.tmp && mv {out}.tmp {out} || rm {out}.tmp', $p);

	return $cmdoutput === "";
}


/**
 * Parse command line options with Usage given as template and return assoc array.
 * @param $usage Usage parsed to determine options, example see below.
 * @param $p['optbeforearg'] Stop parsing options when first argument is encountered
 * @param $p['noexit'] Just return false in case of help or bad usage
 * @return Associative array of options, key 'args' for optional non-option args
 *
 * Usage: myprogram.php [OPTIONS] PATTERN [MOREARGS]
 *  -s, --short      Use short ouput
 *  -f, --file=FILE  Use FILE for input [foo.txt]
 *  -x EXTENSION     Ignore EXTENSION
 * Mandatory arguments from the Usage: line are returned under their (lowercased!) name.
 * Option text must be indented; if long and short option present, value is stored in long one.
 * For long opts with arguments, = is mandatory in usage. Defaults in [ ] at end of line
 * Two or more blanks must be in front of option explanation
 * Options without arguments store true or false under their key
 * Option --help or -h prints de-indented usage and exits (needs not be in usage)
 * Option --debug foo-bar will store bar in EDC('foo'). (needs not be in usage)
 * Option --verbose sets debug variable verbose for EDC('verbose', $bar). (must be in usage)
 */
static function getopt($usage, $p = array())
{
	$GLOBALS['it_stdin'] = array(
		'fd' => null,
		'args' => array(),
		'filename' => "",
		'line' => 0,
	);

	$result = array('args' => array());

	if ($indentation = it::match('^\s+', $usage))
		$usage = it::replace(array($indentation => "\n"), $usage);

	foreach(explode("\n", trim($usage)) as $usageline)
	{
		$shortoptname = $shortoptarg = $longoptname = $longoptarg = "";
		foreach (explode(',', $usageline) as $optdesc)
		{
			$optdesc = trim($optdesc);
			if ($matches = (array)it::match('^--(\w[\w-]*)(=[A-Z])?', $optdesc))
				list($longoptname, $longoptarg) = $matches;
			elseif ($matches = (array)it::match('^-(\w)( [A-Z])?', $optdesc))
				list($shortoptname, $shortoptarg) = $matches;
		}	

		if (($default = it::match('\[(.*)\]\s*$', $usageline)) !== null)
		{
			if ($longoptarg || $shortoptarg)
				$defaults[$longoptname ? $longoptname : $shortoptname] = it::replace(array('^default:\s*' => ""), trim($default));
			else if (($longoptname || $shortoptname) && it::match('^\w+$', $default))
				it::error('defaults for boolean arguments not supported!');
		}

		if ($longoptname || $shortoptname)
		{
			if ($longoptname && ($shortoptname || $shortoptname === "0"))
				$alias[$shortoptname] = $longoptname;

			$witharg[$longoptname ? $longoptname : $shortoptname] = $longoptarg || $shortoptarg;
		}
	}
	$witharg['debug'] = true;

	$mandatoryargs = array();
	if ($tmp = trim(it::replace(array("\n.*" => "", "^\S+\s+\S+\s*" => "", "\[.*?\]\s*" => ""), trim($usage))))
		$mandatoryargs = preg_split('/\s+/', $tmp);
	if ($mandatoryargs && !it::match("Usage:", $usage))
		it::error("Usage string must contain 'Usage:'");

	foreach (array_slice($_SERVER['argv'], 1) as $arg)
	{
		if ($eat)
		{
			if (it::match('^--?\w', $arg))	# Already next option => Missing argument?
				$err = true;
			else
				$result[array_shift($eat)] = $arg;
		}
		else if ($arg == "--")
			$noopts = true;
		elseif (!$noopts && ($matches = (array)it::match('^--(\w[\w-]*)(=.*)?', $arg)))
		{
			list($optname, $val) = $matches;
			if (!isset($witharg[$optname]) || isset($val) && !$witharg[$optname])
				$err = true;
			else if($witharg[$optname] && !$val)
				$eat[] = $optname;
			else
				$result[$optname] = $val ? substr($val, 1) : true;
		}
		else if (!$noopts && (($letters = it::match('^-(\w+)', $arg)) || $letters === "0"))
		{
			foreach (explode("\n", trim(chunk_split($letters, 1, "\n"))) as $letter)
			{
				$optname = $alias[$letter] ? $alias[$letter] : $letter;
				if ($witharg[$optname])
					$eat[] = $optname;
				else if (!isset($witharg[$optname]))
					$err = true;
				else
					$result[$optname] = true;
			}
		}
		elseif($mandatoryargs)
			$result[strtolower(array_shift($mandatoryargs))] = $seenarg = $arg;
		else
			$result['args'][] = $seenarg = $arg;

		if ($p['optbeforearg'] && $seenarg)
			$noopts = true;
	}

	if ($err || $eat || $result['h'] || $result['help'] || $mandatoryargs)
	{
		fputs(($result['h'] || $result['help'] ? STDOUT : STDERR), trim($usage) . "\n");
		return $p['noexit'] ? false : exit(1);
	}

	if ($result['debug'])    
	{
		foreach (preg_split('/[.,]/', $result['debug']) as $ultrad)
		{
			$ultravar = preg_split('/[-=:]/', $ultrad, 2);
			$GLOBALS["debug_$ultravar[0]"] = isset($ultravar[1]) ? $ultravar[1] : 1;
		}
	}

	$GLOBALS['it_stdin']['args'] = $result['args'] ? $result['args'] : array("-");
	it::_stdin_next();

	if ($result['verbose'])
		$GLOBALS['debug_verbose']++;

	return $result + (array)$defaults;
}

static function _stdin_next()
{
	if ($result = $GLOBALS['it_stdin']['args'])
	{
		$GLOBALS['it_stdin']['filename'] = array_shift($GLOBALS['it_stdin']['args']);
		$GLOBALS['it_stdin']['fd'] = ($GLOBALS['it_stdin']['filename'] == "-") ? STDIN : @fopen($GLOBALS['it_stdin']['filename'], "r");
		$GLOBALS['it_stdin']['line'] = 0;
	}

	return $result;
}

/**
 * Get one line from stdin (or files given on command line) a la Perl <>.
 * Note: You need to call getopt() before using this function.
 * @return Line (including newline) or false on eof
 */
static function gets()
{
	do {
		$result = fgets($GLOBALS['it_stdin']['fd']);
	} while (($result === false) && it::_stdin_next());

	$GLOBALS['it_stdin']['line']++;
	return $result;
}

/**
 * Like time() but overridable via debug param
 */
static function time()
{
	return ($t = EDC('time')) ? strtotime(rtrim(it::replace(array('(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)-(\d\d)(-(\d\d))?' => '$1-$2-$3 $4:$5:$7', '^(\d\d)-?(\d\d)(-?(\d\d))?$' => 'today $1:$2:$4'), $t), ":")) : time();
}

/**
 * Output formatted and localized date
 * @param format optional format (default is 2007-01-02 03:04:05).
 *        Other formats are "date", "datetime", "time".
 *        Formats can be qualified with language, e.g. "date:en"
 *        Special formats without language support are "icsdate", "icsdatetime".
 * @param stamp optional unix timestamp (default is now).
 *        If it contains nondigits, it is fed to strtotime
 */
static function date($format = "", $stamp = null)
{
	if (!isset($stamp))
		$stamp = it::time();
	else if (is_string($stamp) && !ctype_digit($stamp))
		$stamp = strtotime($stamp, it::time());

	list($name, $language) = explode(":", $format);

	if ($format && !$language)
		$language = T_lang();

	$formats = array(
		"" => "Y-m-d H:i:s",
		"date" => "d.m.Y",
		"datetime" => "d.m.Y H:i",
		"time" => "H:i",
		"date:en" => "m/d/Y",
		"datetime:en" => "m/d/Y H:i",
		"icsdate" => "Ymd",
	);

	if (!($formatstring = $formats["$name:$language"]) && !($formatstring = $formats[$name]))
	{
		if ($format == "icsdatetime")	# Special icsdate or icsdatetime format? Use UTC time format for Google Calendar to be happy
			return gmdate("Ymd", $stamp) . "T" . gmdate("His", $stamp) . "Z";
		else
			$formatstring = $format;
	}

	return date($formatstring, $stamp);
}

# Internal: Convert expression or funcname or function literal to callable
static function createfunc($code)
{
	if (is_string($code) && it::match('^[\w:]+$', $code) && is_callable($code))
		$code .= '($v)';

	if (!is_callable($func = $code) && !function_exists($func = "_it_map_" . md5($code)))
		eval("function $func(\$k, \$v) { return $code; }");	# Anonymous functions/closures are slower than this (PHP 7.1)

	return $func;
}

/**
 * Iterate over an array, replacing every element by function value or expression
 * @param $code The function (gets key and value as args) or expression as string (gets $k and $v) to apply
 * @param $array The array to iterate over
 * @param $p['keys'] Only modify elements with given keys (keys can be array or comma separated)
 */
static function map($code, $array, $p = null)
{
	$func = self::createfunc($code);
	$result = is_array($array) ? $array : iterator_to_array($array);
	foreach (isset($p['keys']) ? it::filter_keys($array, $p['keys']) : $array as $k => $v)
		$result[$k] = $func($k, $v);

	return (array)$result;
}

/**
 * Return only elements of $rray for which $code argument evaluates to true. Preserves keys
 * @param $code The function (gets key and value as args) or string expression (gets $k and $v) to evaluate
 */
static function filter($code, $array)
{
	$func = self::createfunc($code);
	foreach (is_array($array) ? $array : iterator_to_array($array) as $k => $v)
		if ($func($k, $v))
			$result[$k] = $v;

	return (array)$result;
}


/**
 * Send a mail.
 * @param $p Header => Content (e.g To => me@example.com, Body => bodytext, Html => htmlbodytext)
 * @param $p['forcemail'] Send this mail even if we're on a twin or devel machine
 * @return nothing useful
 */
static function mail($p)
{
	$headers = $p;
	unset($headers['forcemail'], $headers['Body'], $headers['Html']);
	$mail = new it_mail(array_filter($headers));
	$mail->add_body($p['Body']);
	$mail->add_body($p['Html'], IT_MAIL_HTML);

	return $mail->send($p);
}

/**
 * Add an extra md5 based directory name on bottom of path. foo/bar -> foo/07/bar
 */
static function add_dir($path)
{
	return dirname($path) . "/" . substr(md5(basename($path)), 0, 2) . "/" . basename($path);
}

/**
 * Reads a file and returns it as string or in one of several formats. Two params: filename and flags
 * @param $filename name of file to read or - for stdin
 * @param $p['keyval'] each line of the file is one tab-separated key/value pair, return assoc array
 * @param $p['lines'] return file as array of lines without newline
 */
static function file_get($filename, $p = array())
{
	if (($data = file_get_contents($filename == "-" ? "php://stdin" : $filename)) !== false)
	{
		if ($p['keyval'])
		{
			foreach (explode("\n", rtrim($data, "\n")) as $line)
			{
				$arr = explode("\t", $line, 2);
				$result[$arr[0]] = $arr[1];
			}
		}
		else if ($p['lines'])
			$result = explode("\n", rtrim($data, "\n"));
		else
			$result = $data;
	}

	return $result;
}

/**
 * Atomically write data to a file with several serialization modes
 * @param $filename name of file to write or - for stdout
 * @param $data data to write, string by default
 * @param $p['keyval'] $data must be an assoc array and is written as tab-separated lines
 * @param $p['lines'] write array of lines, appending newline
 * @param $p['mkdir'] create parent directory (one level)
 */
static function file_put($filename, $data, $p = array())
{
	if ($p['keyval'])
		$data = join("", it::map('"$k\t$v\n"', $data));
	else if ($p['lines'])
		$data = count((array)$data) ? join("\n", (array)$data) ."\n" : "";

	if ($p['mkdir'] && $filename != "-")
		@mkdir(dirname($filename));

	$tmpfile = dirname($filename) . "/.it_put." . basename($filename) . "." . getmypid();
	if (($result = file_put_contents($filename == "-" ? "php://stdout" : $tmpfile, $data)) !== false && $filename != "-")
			$result = rename($tmpfile, $filename);

	return $result;
}

/**
 * Create an it_pipe from a file
 * @param $fn filename to read. if omitted, stdin is used.
 */
static function cat($fn = null)
{
	return new it_pipe(array('fn' => $fn));
}

/**
 * Create an it_pipe from a command or a lines array
 * @param $cmd command to execute, or if it is an array, the lines to use
 * @param $args named arguments to fill into cmd. see it::exec for usage
 */
static function pipe($cmd, $args = array())
{
	return new it_pipe(is_string($cmd) ? array('cmd' => $cmd, 'args' => $args) : array('data' => $cmd));
}

/**
 * Encodes data to json with unescaped slahes, unescape unicode and on devel pretty printing
 */
static function json_encode($data)
{
	return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | (it::is_devel() ? JSON_PRETTY_PRINT : 0));
}

/**
 * Decodes json data and provides warning if failed
 */
static function json_decode($json)
{
	return ($data = json_decode($json)) === null && $json != 'null' ? it::error(array('title' => "invalid json", 'body' => $json)) : $data;
}

/**
 * Retuns sorted array. mode string can contain combinations of a for assoc, k for key, r for reverse and n for numeric
 */
static function sort($array, $mode = "")
{
	$func = it::replace(array('n' => ""), count_chars($mode, 3)) . "sort"; # count_chars sorts flags
	$func($array, it::match('n', $mode) ? SORT_NUMERIC : 0);

	return $array;
}

/**
 * returns $a modulo $n as a positive number between 0 and $n - 1
 */
static function mod($a, $n)
{
	return (($a % $n) + $n) % $n;
}

}