From 3e194c79060b84d04ce0b2c577ea4b68c7d8d732 Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Sat, 11 Nov 2023 17:12:00 +0100 Subject: [PATCH 01/32] Domains datatable - server side processing Signed-off-by: Kristian Feldsam --- data/web/inc/lib/ssp.class.php | 587 +++++++++++++++++++++++++++++++++ data/web/js/site/mailbox.js | 33 +- data/web/json_api.php | 65 +++- 3 files changed, 666 insertions(+), 19 deletions(-) create mode 100644 data/web/inc/lib/ssp.class.php diff --git a/data/web/inc/lib/ssp.class.php b/data/web/inc/lib/ssp.class.php new file mode 100644 index 000000000..c36c627cb --- /dev/null +++ b/data/web/inc/lib/ssp.class.php @@ -0,0 +1,587 @@ + 'utf8'` - you might need this depending on your PHP / MySQL config + * @return resource PDO connection + */ + static function db ( $conn ) + { + if ( is_array( $conn ) ) { + return self::sql_connect( $conn ); + } + + return $conn; + } + + + /** + * Paging + * + * Construct the LIMIT clause for server-side processing SQL query + * + * @param array $request Data sent to server by DataTables + * @param array $columns Column information array + * @return string SQL limit clause + */ + static function limit ( $request, $columns ) + { + $limit = ''; + + if ( isset($request['start']) && $request['length'] != -1 ) { + $limit = "LIMIT ".intval($request['start']).", ".intval($request['length']); + } + + return $limit; + } + + + /** + * Ordering + * + * Construct the ORDER BY clause for server-side processing SQL query + * + * @param array $request Data sent to server by DataTables + * @param array $columns Column information array + * @return string SQL order by clause + */ + static function order ( $tableAS, $request, $columns ) + { + $order = ''; + + if ( isset($request['order']) && count($request['order']) ) { + $orderBy = array(); + $dtColumns = self::pluck( $columns, 'dt' ); + + for ( $i=0, $ien=count($request['order']) ; $i<$ien ; $i++ ) { + // Convert the column index into the column data property + $columnIdx = intval($request['order'][$i]['column']); + $requestColumn = $request['columns'][$columnIdx]; + + $columnIdx = array_search( $columnIdx, $dtColumns ); + $column = $columns[ $columnIdx ]; + + if ( $requestColumn['orderable'] == 'true' ) { + $dir = $request['order'][$i]['dir'] === 'asc' ? + 'ASC' : + 'DESC'; + + $orderBy[] = '`'.$tableAS.'`.`'.$column['db'].'` '.$dir; + } + } + + if ( count( $orderBy ) ) { + $order = 'ORDER BY '.implode(', ', $orderBy); + } + } + + return $order; + } + + + /** + * Searching / Filtering + * + * Construct the WHERE clause for server-side processing SQL query. + * + * NOTE this does not match the built-in DataTables filtering which does it + * word by word on any field. It's possible to do here performance on large + * databases would be very poor + * + * @param array $request Data sent to server by DataTables + * @param array $columns Column information array + * @param array $bindings Array of values for PDO bindings, used in the + * sql_exec() function + * @return string SQL where clause + */ + static function filter ( $tablesAS, $request, $columns, &$bindings ) + { + $globalSearch = array(); + $columnSearch = array(); + $dtColumns = self::pluck( $columns, 'dt' ); + + if ( isset($request['search']) && $request['search']['value'] != '' ) { + $str = $request['search']['value']; + + for ( $i=0, $ien=count($request['columns']) ; $i<$ien ; $i++ ) { + $requestColumn = $request['columns'][$i]; + $columnIdx = array_search( $requestColumn['data'], $dtColumns ); + $column = $columns[ $columnIdx ]; + + if ( $requestColumn['searchable'] == 'true' ) { + if(!empty($column['db'])){ + $binding = self::bind( $bindings, '%'.$str.'%', PDO::PARAM_STR ); + $globalSearch[] = "`".$tablesAS."`.`".$column['db']."` LIKE ".$binding; + } + } + } + } + + // Individual column filtering + if ( isset( $request['columns'] ) ) { + for ( $i=0, $ien=count($request['columns']) ; $i<$ien ; $i++ ) { + $requestColumn = $request['columns'][$i]; + $columnIdx = array_search( $requestColumn['data'], $dtColumns ); + $column = $columns[ $columnIdx ]; + + $str = $requestColumn['search']['value']; + + if ( $requestColumn['searchable'] == 'true' && + $str != '' ) { + if(!empty($column['db'])){ + $binding = self::bind( $bindings, '%'.$str.'%', PDO::PARAM_STR ); + $columnSearch[] = "`".$tablesAS."`.`".$column['db']."` LIKE ".$binding; + } + } + } + } + + // Combine the filters into a single string + $where = ''; + + if ( count( $globalSearch ) ) { + $where = '('.implode(' OR ', $globalSearch).')'; + } + + if ( count( $columnSearch ) ) { + $where = $where === '' ? + implode(' AND ', $columnSearch) : + $where .' AND '. implode(' AND ', $columnSearch); + } + + if ( $where !== '' ) { + $where = 'WHERE '.$where; + } + + return $where; + } + + + /** + * Perform the SQL queries needed for an server-side processing requested, + * utilising the helper functions of this class, limit(), order() and + * filter() among others. The returned array is ready to be encoded as JSON + * in response to an SSP request, or can be modified if needed before + * sending back to the client. + * + * @param array $request Data sent to server by DataTables + * @param array|PDO $conn PDO connection resource or connection parameters array + * @param string $table SQL table to query + * @param string $primaryKey Primary key of the table + * @param array $columns Column information array + * @return array Server-side processing response array + */ + static function simple ( $request, $conn, $table, $primaryKey, $columns ) + { + $bindings = array(); + $db = self::db( $conn ); + + // Allow for a JSON string to be passed in + if (isset($request['json'])) { + $request = json_decode($request['json'], true); + } + + // table AS + $tablesAS = null; + if(is_array($table)) { + $tablesAS = $table[1]; + $table = $table[0]; + } + + // Build the SQL query string from the request + $limit = self::limit( $request, $columns ); + $order = self::order( $tablesAS, $request, $columns ); + $where = self::filter( $tablesAS, $request, $columns, $bindings ); + + // Main query to actually get the data + $data = self::sql_exec( $db, $bindings, + "SELECT `$tablesAS`.`".implode("`, `$tablesAS`.`", self::pluck($columns, 'db'))."` + FROM `$table` AS `$tablesAS` + $where + $order + $limit" + ); + + // Data set length after filtering + $resFilterLength = self::sql_exec( $db, $bindings, + "SELECT COUNT(`{$primaryKey}`) + FROM `$table` AS `$tablesAS` + $where" + ); + $recordsFiltered = $resFilterLength[0][0]; + + // Total data set length + $resTotalLength = self::sql_exec( $db, + "SELECT COUNT(`{$primaryKey}`) + FROM `$table` AS `$tablesAS`" + ); + $recordsTotal = $resTotalLength[0][0]; + + /* + * Output + */ + return array( + "draw" => isset ( $request['draw'] ) ? + intval( $request['draw'] ) : + 0, + "recordsTotal" => intval( $recordsTotal ), + "recordsFiltered" => intval( $recordsFiltered ), + "data" => self::data_output( $columns, $data ) + ); + } + + + /** + * The difference between this method and the `simple` one, is that you can + * apply additional `where` conditions to the SQL queries. These can be in + * one of two forms: + * + * * 'Result condition' - This is applied to the result set, but not the + * overall paging information query - i.e. it will not effect the number + * of records that a user sees they can have access to. This should be + * used when you want apply a filtering condition that the user has sent. + * * 'All condition' - This is applied to all queries that are made and + * reduces the number of records that the user can access. This should be + * used in conditions where you don't want the user to ever have access to + * particular records (for example, restricting by a login id). + * + * In both cases the extra condition can be added as a simple string, or if + * you are using external values, as an assoc. array with `condition` and + * `bindings` parameters. The `condition` is a string with the SQL WHERE + * condition and `bindings` is an assoc. array of the binding names and + * values. + * + * @param array $request Data sent to server by DataTables + * @param array|PDO $conn PDO connection resource or connection parameters array + * @param string|array $table SQL table to query, if array second key is AS + * @param string $primaryKey Primary key of the table + * @param array $columns Column information array + * @param string $join JOIN sql string + * @param string|array $whereResult WHERE condition to apply to the result set + * @return array Server-side processing response array + */ + static function complex ( + $request, + $conn, + $table, + $primaryKey, + $columns, + $join=null, + $whereResult=null + ) { + $bindings = array(); + $db = self::db( $conn ); + + // table AS + $tablesAS = null; + if(is_array($table)) { + $tablesAS = $table[1]; + $table = $table[0]; + } + + // Build the SQL query string from the request + $limit = self::limit( $request, $columns ); + $order = self::order( $tablesAS, $request, $columns ); + $where = self::filter( $tablesAS, $request, $columns, $bindings ); + + // whereResult can be a simple string, or an assoc. array with a + // condition and bindings + if ( $whereResult ) { + $str = $whereResult; + + if ( is_array($whereResult) ) { + $str = $whereResult['condition']; + + if ( isset($whereResult['bindings']) ) { + self::add_bindings($bindings, $whereResult); + } + } + + $where = $where ? + $where .' AND '.$str : + 'WHERE '.$str; + } + + // Main query to actually get the data + $data = self::sql_exec( $db, $bindings, + "SELECT `$tablesAS`.`".implode("`, `$tablesAS`.`", self::pluck($columns, 'db'))."` + FROM `$table` AS `$tablesAS` + $join + $where + $order + $limit" + ); + + // Data set length after filtering + $resFilterLength = self::sql_exec( $db, $bindings, + "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) + FROM `$table` AS `$tablesAS` + $join + $where" + ); + $recordsFiltered = $resFilterLength[0][0]; + + // Total data set length + $resTotalLength = self::sql_exec( $db, $bindings, + "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) + FROM `$table` AS `$tablesAS` + $join + $where" + ); + $recordsTotal = $resTotalLength[0][0]; + + /* + * Output + */ + return array( + "draw" => isset ( $request['draw'] ) ? + intval( $request['draw'] ) : + 0, + "recordsTotal" => intval( $recordsTotal ), + "recordsFiltered" => intval( $recordsFiltered ), + "data" => self::data_output( $columns, $data ) + ); + } + + + /** + * Connect to the database + * + * @param array $sql_details SQL server connection details array, with the + * properties: + * * host - host name + * * db - database name + * * user - user name + * * pass - user password + * @return resource Database connection handle + */ + static function sql_connect ( $sql_details ) + { + try { + $db = @new PDO( + "mysql:host={$sql_details['host']};dbname={$sql_details['db']}", + $sql_details['user'], + $sql_details['pass'], + array( PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION ) + ); + } + catch (PDOException $e) { + self::fatal( + "An error occurred while connecting to the database. ". + "The error reported by the server was: ".$e->getMessage() + ); + } + + return $db; + } + + + /** + * Execute an SQL query on the database + * + * @param resource $db Database handler + * @param array $bindings Array of PDO binding values from bind() to be + * used for safely escaping strings. Note that this can be given as the + * SQL query string if no bindings are required. + * @param string $sql SQL query to execute. + * @return array Result from the query (all rows) + */ + static function sql_exec ( $db, $bindings, $sql=null ) + { + // Argument shifting + if ( $sql === null ) { + $sql = $bindings; + } + + $stmt = $db->prepare( $sql ); + + // Bind parameters + if ( is_array( $bindings ) ) { + for ( $i=0, $ien=count($bindings) ; $i<$ien ; $i++ ) { + $binding = $bindings[$i]; + $stmt->bindValue( $binding['key'], $binding['val'], $binding['type'] ); + } + } + + // Execute + try { + $stmt->execute(); + } + catch (PDOException $e) { + self::fatal( "An SQL error occurred: ".$e->getMessage() ); + } + + // Return all + return $stmt->fetchAll( PDO::FETCH_BOTH ); + } + + + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Internal methods + */ + + /** + * Throw a fatal error. + * + * This writes out an error message in a JSON string which DataTables will + * see and show to the user in the browser. + * + * @param string $msg Message to send to the client + */ + static function fatal ( $msg ) + { + echo json_encode( array( + "error" => $msg + ) ); + + exit(0); + } + + /** + * Create a PDO binding key which can be used for escaping variables safely + * when executing a query with sql_exec() + * + * @param array &$a Array of bindings + * @param * $val Value to bind + * @param int $type PDO field type + * @return string Bound key to be used in the SQL where this parameter + * would be used. + */ + static function bind ( &$a, $val, $type ) + { + $key = ':binding_'.count( $a ); + + $a[] = array( + 'key' => $key, + 'val' => $val, + 'type' => $type + ); + + return $key; + } + + static function add_bindings(&$bindings, $vals) + { + foreach($vals['bindings'] as $key => $value) { + $bindings[] = array( + 'key' => $key, + 'val' => $value, + 'type' => PDO::PARAM_STR + ); + } + } + + + /** + * Pull a particular property from each assoc. array in a numeric array, + * returning and array of the property values from each item. + * + * @param array $a Array to get data from + * @param string $prop Property to read + * @return array Array of property values + */ + static function pluck ( $a, $prop ) + { + $out = array(); + + for ( $i=0, $len=count($a) ; $i<$len ; $i++ ) { + if ( empty($a[$i][$prop]) && $a[$i][$prop] !== 0 ) { + continue; + } + + //removing the $out array index confuses the filter method in doing proper binding, + //adding it ensures that the array data are mapped correctly + $out[$i] = $a[$i][$prop]; + } + + return $out; + } + + + /** + * Return a string from an array or a string + * + * @param array|string $a Array to join + * @param string $join Glue for the concatenation + * @return string Joined string + */ + static function _flatten ( $a, $join = ' AND ' ) + { + if ( ! $a ) { + return ''; + } + else if ( $a && is_array($a) ) { + return implode( $join, $a ); + } + return $a; + } +} + diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index c2b1761de..5e1ba3151 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -435,7 +435,7 @@ jQuery(function($){ var table = $('#domain_table').DataTable({ responsive: true, processing: true, - serverSide: false, + serverSide: true, stateSave: true, pageLength: pagination_size, dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" + @@ -447,9 +447,9 @@ jQuery(function($){ }, ajax: { type: "GET", - url: "/api/v1/get/domain/all", + url: "/api/v1/get/domain/datatables", dataSrc: function(json){ - $.each(json, function(i, item) { + $.each(json.data, function(i, item) { item.domain_name = escapeHtml(item.domain_name); item.aliases = item.aliases_in_domain + " / " + item.max_num_aliases_for_domain; @@ -498,7 +498,7 @@ jQuery(function($){ } }); - return json; + return json.data; } }, columns: [ @@ -528,17 +528,20 @@ jQuery(function($){ { title: lang.aliases, data: 'aliases', + searchable: false, defaultContent: '' }, { title: lang.mailboxes, data: 'mailboxes', + searchable: false, responsivePriority: 4, defaultContent: '' }, { title: lang.domain_quota, data: 'quota', + searchable: false, defaultContent: '', render: function (data, type) { data = data.split("/"); @@ -548,6 +551,8 @@ jQuery(function($){ { title: lang.stats, data: 'stats', + searchable: false, + orderable: false, defaultContent: '', render: function (data, type) { data = data.split("/"); @@ -557,21 +562,29 @@ jQuery(function($){ { title: lang.mailbox_defquota, data: 'def_quota_for_mbox', + searchable: false, + orderable: false, defaultContent: '' }, { title: lang.mailbox_quota, data: 'max_quota_for_mbox', + searchable: false, + orderable: false, defaultContent: '' }, { title: 'RL', data: 'rl', + searchable: false, + orderable: false, defaultContent: '' }, { title: lang.backup_mx, data: 'backupmx', + searchable: false, + orderable: false, defaultContent: '', redner: function (data, type){ return 1==value ? '' : 0==value && ''; @@ -580,30 +593,40 @@ jQuery(function($){ { title: lang.domain_admins, data: 'domain_admins', + searchable: false, + orderable: false, defaultContent: '', className: 'none' }, { title: lang.created_on, data: 'created', + searchable: false, + orderable: false, defaultContent: '', className: 'none' }, { title: lang.last_modified, data: 'modified', + searchable: false, + orderable: false, defaultContent: '', className: 'none' }, { title: 'Tags', data: 'tags', + searchable: false, + orderable: false, defaultContent: '', className: 'none' }, { title: lang.active, data: 'active', + searchable: false, + orderable: false, defaultContent: '', responsivePriority: 6, render: function (data, type) { @@ -613,6 +636,8 @@ jQuery(function($){ { title: lang.action, data: 'action', + searchable: false, + orderable: false, className: 'dt-sm-head-hidden dt-data-w100 dtr-col-md dt-text-right', responsivePriority: 5, defaultContent: '' diff --git a/data/web/json_api.php b/data/web/json_api.php index b375bc8e4..6c008a604 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -15,7 +15,7 @@ function api_log($_data) { continue; } - $value = json_decode($value, true); + $value = json_decode($value, true); if ($value) { if (is_array($value)) unset($value["csrf_token"]); foreach ($value as $key => &$val) { @@ -23,7 +23,7 @@ function api_log($_data) { $val = '*'; } } - $value = json_encode($value); + $value = json_encode($value); } $data_var[] = $data . "='" . $value . "'"; } @@ -44,7 +44,7 @@ function api_log($_data) { 'msg' => 'Redis: '.$e ); return false; - } + } } if (isset($_GET['query'])) { @@ -178,12 +178,12 @@ if (isset($_GET['query'])) { // parse post data $post = trim(file_get_contents('php://input')); if ($post) $post = json_decode($post); - + // process registration data from authenticator try { // decode base64 strings $clientDataJSON = base64_decode($post->clientDataJSON); - $attestationObject = base64_decode($post->attestationObject); + $attestationObject = base64_decode($post->attestationObject); // processCreate($clientDataJSON, $attestationObject, $challenge, $requireUserVerification=false, $requireUserPresent=true, $failIfRootMismatch=true) $data = $WebAuthn->processCreate($clientDataJSON, $attestationObject, $_SESSION['challenge'], false, true); @@ -250,7 +250,7 @@ if (isset($_GET['query'])) { default: process_add_return(mailbox('add', 'domain', $attr)); break; - } + } break; case "resource": process_add_return(mailbox('add', 'resource', $attr)); @@ -470,7 +470,7 @@ if (isset($_GET['query'])) { // false, if only internal is allowed // null, if internal and cross-platform is allowed $createArgs = $WebAuthn->getCreateArgs($_SESSION["mailcow_cc_username"], $_SESSION["mailcow_cc_username"], $_SESSION["mailcow_cc_username"], 30, false, $GLOBALS['WEBAUTHN_UV_FLAG_REGISTER'], null, $excludeCredentialIds); - + print(json_encode($createArgs)); $_SESSION['challenge'] = $WebAuthn->getChallenge(); return; @@ -523,9 +523,44 @@ if (isset($_GET['query'])) { case "domain": switch ($object) { + case "datatables": + $table = ['domain', 'd']; + $primaryKey = 'domain'; + $columns = [ + ['db' => 'domain', 'dt' => 2], + ['db' => 'aliases', 'dt' => 3], + ['db' => 'mailboxes', 'dt' => 4], + ['db' => 'quota', 'dt' => 5], + ]; + + require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/lib/ssp.class.php'; + global $pdo; + if($_SESSION['mailcow_cc_role'] === 'admin') { + $data = SSP::simple($_GET, $pdo, $table, $primaryKey, $columns); + } elseif ($_SESSION['mailcow_cc_role'] === 'domainadmin') { + $data = SSP::complex($_GET, $pdo, $table, $primaryKey, $columns, + 'INNER JOIN domain_admins as da ON da.domain = d.domain', + [ + 'condition' => 'da.active = 1 and da.username = :username', + 'bindings' => ['username' => $_SESSION['mailcow_cc_username']] + ]); + } + + if (!empty($data['data'])) { + $domainsData = []; + foreach ($data['data'] as $domain) { + if ($details = mailbox('get', 'domain_details', $domain[2])) { + $domainsData[] = $details; + } + } + $data['data'] = $domainsData; + } + + process_get_return($data); + break; case "all": $tags = null; - if (isset($_GET['tags']) && $_GET['tags'] != '') + if (isset($_GET['tags']) && $_GET['tags'] != '') $tags = explode(',', $_GET['tags']); $domains = mailbox('get', 'domains', null, $tags); @@ -1014,7 +1049,7 @@ if (isset($_GET['query'])) { case "all": case "reduced": $tags = null; - if (isset($_GET['tags']) && $_GET['tags'] != '') + if (isset($_GET['tags']) && $_GET['tags'] != '') $tags = explode(',', $_GET['tags']); if (empty($extra)) $domains = mailbox('get', 'domains'); @@ -1048,7 +1083,7 @@ if (isset($_GET['query'])) { break; default: $tags = null; - if (isset($_GET['tags']) && $_GET['tags'] != '') + if (isset($_GET['tags']) && $_GET['tags'] != '') $tags = explode(',', $_GET['tags']); if ($tags === null) { @@ -1058,7 +1093,7 @@ if (isset($_GET['query'])) { $mailboxes = mailbox('get', 'mailboxes', $object, $tags); if (is_array($mailboxes)) { foreach ($mailboxes as $mailbox) { - if ($details = mailbox('get', 'mailbox_details', $mailbox)) + if ($details = mailbox('get', 'mailbox_details', $mailbox)) $data[] = $details; } } @@ -1557,15 +1592,15 @@ if (isset($_GET['query'])) { 'solr_size' => $solr_size, 'solr_documents' => $solr_documents )); - break; + break; case "host": if (!$extra){ $stats = docker("host_stats"); echo json_encode($stats); - } + } else if ($extra == "ip") { // get public ips - + $curl = curl_init(); curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); curl_setopt($curl, CURLOPT_POST, 0); @@ -1972,7 +2007,7 @@ if (isset($_GET['query'])) { exit(); } } -if ($_SESSION['mailcow_cc_api'] === true) { +if (array_key_exists('mailcow_cc_api', $_SESSION) && $_SESSION['mailcow_cc_api'] === true) { if (isset($_SESSION['mailcow_cc_api']) && $_SESSION['mailcow_cc_api'] === true) { unset($_SESSION['return']); } From 28cec9969931de92b128c086b1f113c35e600210 Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Sun, 12 Nov 2023 10:26:38 +0100 Subject: [PATCH 02/32] Mailboxes datatable - server side processing Signed-off-by: Kristian Feldsam --- data/web/js/site/mailbox.js | 39 ++++++++++++++++++------------------- data/web/json_api.php | 33 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index 5e1ba3151..fe6113a8f 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -869,7 +869,7 @@ jQuery(function($){ var table = $('#mailbox_table').DataTable({ responsive: true, processing: true, - serverSide: false, + serverSide: true, stateSave: true, pageLength: pagination_size, dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" + @@ -878,13 +878,12 @@ jQuery(function($){ language: lang_datatables, initComplete: function(settings, json){ hideTableExpandCollapseBtn('#tab-mailboxes', '#mailbox_table'); - filterByDomain(json, 8, table); }, ajax: { type: "GET", - url: "/api/v1/get/mailbox/reduced", + url: "/api/v1/get/mailbox/datatables", dataSrc: function(json){ - $.each(json, function (i, item) { + $.each(json.data, function (i, item) { item.quota = { sortBy: item.quota_used, value: item.quota @@ -970,7 +969,7 @@ jQuery(function($){ } }); - return json; + return json.data; } }, columns: [ @@ -1000,13 +999,15 @@ jQuery(function($){ { title: lang.domain_quota, data: 'quota.value', + searchable: false, responsivePriority: 8, - defaultContent: '', - orderData: 23 + defaultContent: '' }, { title: lang.last_mail_login, data: 'last_mail_login', + searchable: false, + orderable: false, defaultContent: '', responsivePriority: 7, render: function (data, type) { @@ -1019,11 +1020,15 @@ jQuery(function($){ { title: lang.last_pw_change, data: 'last_pw_change', + searchable: false, + orderable: false, defaultContent: '' }, { title: lang.in_use, data: 'in_use.value', + searchable: false, + orderable: false, defaultContent: '', responsivePriority: 9, className: 'dt-data-w100', @@ -1092,6 +1097,8 @@ jQuery(function($){ { title: lang.msg_num, data: 'messages', + searchable: false, + orderable: false, defaultContent: '', responsivePriority: 5 }, @@ -1116,6 +1123,8 @@ jQuery(function($){ { title: lang.active, data: 'active', + searchable: false, + orderable: false, defaultContent: '', responsivePriority: 4, render: function (data, type) { @@ -1125,22 +1134,12 @@ jQuery(function($){ { title: lang.action, data: 'action', + searchable: false, + orderable: false, className: 'dt-sm-head-hidden dt-data-w100 dtr-col-md dt-text-right', responsivePriority: 6, defaultContent: '' - }, - { - title: "", - data: 'quota.sortBy', - defaultContent: '', - className: "d-none" - }, - { - title: "", - data: 'in_use.sortBy', - defaultContent: '', - className: "d-none" - }, + } ] }); diff --git a/data/web/json_api.php b/data/web/json_api.php index 6c008a604..3077bf7b0 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -1046,6 +1046,39 @@ if (isset($_GET['query'])) { break; case "mailbox": switch ($object) { + case "datatables": + $table = ['mailbox', 'm']; + $primaryKey = 'username'; + $columns = [ + ['db' => 'username', 'dt' => 2], + ['db' => 'quota', 'dt' => 3], + ]; + + require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/lib/ssp.class.php'; + global $pdo; + if($_SESSION['mailcow_cc_role'] === 'admin') { + $data = SSP::complex($_GET, $pdo, $table, $primaryKey, $columns, null, "(`m`.`kind` = '' OR `m`.`kind` = NULL)"); + } elseif ($_SESSION['mailcow_cc_role'] === 'domainadmin') { + $data = SSP::complex($_GET, $pdo, $table, $primaryKey, $columns, + 'INNER JOIN domain_admins as da ON da.domain = m.domain', + [ + 'condition' => "(`m`.`kind` = '' OR `m`.`kind` = NULL) AND `da`.`active` = 1 AND `da`.`username` = :username", + 'bindings' => ['username' => $_SESSION['mailcow_cc_username']] + ]); + } + + if (!empty($data['data'])) { + $mailboxData = []; + foreach ($data['data'] as $mailbox) { + if ($details = mailbox('get', 'mailbox_details', $mailbox[2])) { + $mailboxData[] = $details; + } + } + $data['data'] = $mailboxData; + } + + process_get_return($data); + break; case "all": case "reduced": $tags = null; From 4f109c1a9401582264353dc7ac9aa59fb0243ef0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 12 Nov 2023 17:28:57 +0000 Subject: [PATCH 03/32] Update dependency krakjoe/apcu to v5.1.23 Signed-off-by: milkmaker --- data/Dockerfiles/phpfpm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/phpfpm/Dockerfile b/data/Dockerfiles/phpfpm/Dockerfile index 490310336..123061a52 100644 --- a/data/Dockerfiles/phpfpm/Dockerfile +++ b/data/Dockerfiles/phpfpm/Dockerfile @@ -2,7 +2,7 @@ FROM php:8.2-fpm-alpine3.17 LABEL maintainer "The Infrastructure Company GmbH " # renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?.*)$ -ARG APCU_PECL_VERSION=5.1.22 +ARG APCU_PECL_VERSION=5.1.23 # renovate: datasource=github-tags depName=Imagick/imagick versioning=semver-coerced extractVersion=(?.*)$ ARG IMAGICK_PECL_VERSION=3.7.0 # renovate: datasource=github-tags depName=php/pecl-mail-mailparse versioning=semver-coerced extractVersion=^v(?.*)$ From a8dfa951268169bc9017165e2f6b0b46a30aaf6e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:03:02 +0000 Subject: [PATCH 04/32] Update dependency phpredis/phpredis to v6.0.2 Signed-off-by: milkmaker --- data/Dockerfiles/phpfpm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/phpfpm/Dockerfile b/data/Dockerfiles/phpfpm/Dockerfile index 490310336..88e5c9905 100644 --- a/data/Dockerfiles/phpfpm/Dockerfile +++ b/data/Dockerfiles/phpfpm/Dockerfile @@ -10,7 +10,7 @@ ARG MAILPARSE_PECL_VERSION=3.1.6 # renovate: datasource=github-tags depName=php-memcached-dev/php-memcached versioning=semver-coerced extractVersion=^v(?.*)$ ARG MEMCACHED_PECL_VERSION=3.2.0 # renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?.*)$ -ARG REDIS_PECL_VERSION=6.0.1 +ARG REDIS_PECL_VERSION=6.0.2 # renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?.*)$ ARG COMPOSER_VERSION=2.6.5 From d4dd1e37ce02f264639434e9104a3bf742195c2c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:03:09 +0000 Subject: [PATCH 05/32] Update dependency tianon/gosu to v1.17 Signed-off-by: milkmaker --- data/Dockerfiles/dovecot/Dockerfile | 2 +- data/Dockerfiles/sogo/Dockerfile | 2 +- data/Dockerfiles/solr/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index 2ace9029e..d1413b2ae 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -5,7 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive # renovate: datasource=github-tags depName=dovecot/core versioning=semver-coerced extractVersion=(?.*)$ ARG DOVECOT=2.3.21 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=(?.*)$ -ARG GOSU_VERSION=1.16 +ARG GOSU_VERSION=1.17 ENV LC_ALL C diff --git a/data/Dockerfiles/sogo/Dockerfile b/data/Dockerfiles/sogo/Dockerfile index e8a7410f5..cbc5c93a9 100644 --- a/data/Dockerfiles/sogo/Dockerfile +++ b/data/Dockerfiles/sogo/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer "The Infrastructure Company GmbH " ARG DEBIAN_FRONTEND=noninteractive ARG SOGO_DEBIAN_REPOSITORY=http://packages.sogo.nu/nightly/5/debian/ # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?.*)$ -ARG GOSU_VERSION=1.16 +ARG GOSU_VERSION=1.17 ENV LC_ALL C # Prerequisites diff --git a/data/Dockerfiles/solr/Dockerfile b/data/Dockerfiles/solr/Dockerfile index a6359876b..429133519 100644 --- a/data/Dockerfiles/solr/Dockerfile +++ b/data/Dockerfiles/solr/Dockerfile @@ -3,7 +3,7 @@ FROM solr:7.7-slim USER root # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=(?.*)$ -ARG GOSU_VERSION=1.16 +ARG GOSU_VERSION=1.17 COPY solr.sh / COPY solr-config-7.7.0.xml / From 4dad0002cd72a6464f6193bd51092293a897078b Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Mon, 4 Dec 2023 13:41:09 +0100 Subject: [PATCH 06/32] Domains datatable - server side processing ordering Signed-off-by: Kristian Feldsam --- data/web/inc/functions.mailbox.inc.php | 1 - data/web/inc/lib/ssp.class.php | 30 ++++++++++++++++++++------ data/web/js/site/mailbox.js | 9 ++------ data/web/json_api.php | 11 +++++++--- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index 68cb50f1b..c35294354 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -4323,7 +4323,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $mailboxdata['active'] = $row['active']; $mailboxdata['active_int'] = $row['active']; $mailboxdata['domain'] = $row['domain']; - $mailboxdata['relayhost'] = $row['relayhost']; $mailboxdata['name'] = $row['name']; $mailboxdata['local_part'] = $row['local_part']; $mailboxdata['quota'] = $row['quota']; diff --git a/data/web/inc/lib/ssp.class.php b/data/web/inc/lib/ssp.class.php index c36c627cb..a06e7d09e 100644 --- a/data/web/inc/lib/ssp.class.php +++ b/data/web/inc/lib/ssp.class.php @@ -43,7 +43,7 @@ class SSP { } } else { - if(!empty($column['db'])){ + if(!empty($column['db']) && (!isset($column['dummy']) || $column['dummy'] !== true)){ $row[ $column['dt'] ] = $data[$i][ $columns[$j]['db'] ]; } else{ @@ -115,10 +115,12 @@ class SSP { */ static function order ( $tableAS, $request, $columns ) { + $select = ''; $order = ''; if ( isset($request['order']) && count($request['order']) ) { - $orderBy = array(); + $selects = []; + $orderBy = []; $dtColumns = self::pluck( $columns, 'dt' ); for ( $i=0, $ien=count($request['order']) ; $i<$ien ; $i++ ) { @@ -133,17 +135,26 @@ class SSP { $dir = $request['order'][$i]['dir'] === 'asc' ? 'ASC' : 'DESC'; - - $orderBy[] = '`'.$tableAS.'`.`'.$column['db'].'` '.$dir; + + if(isset($column['order_subquery'])) { + $selects[] = '('.$column['order_subquery'].') AS `'.$column['db'].'_count`'; + $orderBy[] = '`'.$column['db'].'_count` '.$dir; + } else { + $orderBy[] = '`'.$tableAS.'`.`'.$column['db'].'` '.$dir; + } } } + if ( count( $selects ) ) { + $select = ', '.implode(', ', $selects); + } + if ( count( $orderBy ) ) { $order = 'ORDER BY '.implode(', ', $orderBy); } } - return $order; + return [$select, $order]; } @@ -257,13 +268,14 @@ class SSP { } // Build the SQL query string from the request + list($select, $order) = self::order( $tablesAS, $request, $columns ); $limit = self::limit( $request, $columns ); - $order = self::order( $tablesAS, $request, $columns ); $where = self::filter( $tablesAS, $request, $columns, $bindings ); // Main query to actually get the data $data = self::sql_exec( $db, $bindings, "SELECT `$tablesAS`.`".implode("`, `$tablesAS`.`", self::pluck($columns, 'db'))."` + $select FROM `$table` AS `$tablesAS` $where $order @@ -348,8 +360,8 @@ class SSP { } // Build the SQL query string from the request + list($select, $order) = self::order( $tablesAS, $request, $columns ); $limit = self::limit( $request, $columns ); - $order = self::order( $tablesAS, $request, $columns ); $where = self::filter( $tablesAS, $request, $columns, $bindings ); // whereResult can be a simple string, or an assoc. array with a @@ -373,6 +385,7 @@ class SSP { // Main query to actually get the data $data = self::sql_exec( $db, $bindings, "SELECT `$tablesAS`.`".implode("`, `$tablesAS`.`", self::pluck($columns, 'db'))."` + $select FROM `$table` AS `$tablesAS` $join $where @@ -556,6 +569,9 @@ class SSP { if ( empty($a[$i][$prop]) && $a[$i][$prop] !== 0 ) { continue; } + if ( $prop == 'db' && isset($a[$i]['dummy']) && $a[$i]['dummy'] === true ) { + continue; + } //removing the $out array index confuses the filter method in doing proper binding, //adding it ensures that the array data are mapped correctly diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index fe6113a8f..c6f57d457 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -552,7 +552,6 @@ jQuery(function($){ title: lang.stats, data: 'stats', searchable: false, - orderable: false, defaultContent: '', render: function (data, type) { data = data.split("/"); @@ -563,14 +562,12 @@ jQuery(function($){ title: lang.mailbox_defquota, data: 'def_quota_for_mbox', searchable: false, - orderable: false, defaultContent: '' }, { title: lang.mailbox_quota, data: 'max_quota_for_mbox', searchable: false, - orderable: false, defaultContent: '' }, { @@ -584,10 +581,9 @@ jQuery(function($){ title: lang.backup_mx, data: 'backupmx', searchable: false, - orderable: false, defaultContent: '', - redner: function (data, type){ - return 1==value ? '' : 0==value && ''; + render: function (data, type){ + return 1==data ? '' : 0==data && ''; } }, { @@ -626,7 +622,6 @@ jQuery(function($){ title: lang.active, data: 'active', searchable: false, - orderable: false, defaultContent: '', responsivePriority: 6, render: function (data, type) { diff --git a/data/web/json_api.php b/data/web/json_api.php index 3077bf7b0..0f0c23b15 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -528,9 +528,14 @@ if (isset($_GET['query'])) { $primaryKey = 'domain'; $columns = [ ['db' => 'domain', 'dt' => 2], - ['db' => 'aliases', 'dt' => 3], - ['db' => 'mailboxes', 'dt' => 4], - ['db' => 'quota', 'dt' => 5], + ['db' => 'aliases', 'dt' => 3, 'order_subquery' => "SELECT COUNT(*) FROM `alias` WHERE (`domain`= `d`.`domain` OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = `d`.`domain`)) AND `address` NOT IN (SELECT `username` FROM `mailbox`)"], + ['db' => 'mailboxes', 'dt' => 4, 'order_subquery' => "SELECT COUNT(*) FROM `mailbox` WHERE `mailbox`.`domain` = `d`.`domain` AND (`mailbox`.`kind` = '' OR `mailbox`.`kind` = NULL)"], + ['db' => 'quota', 'dt' => 5, 'order_subquery' => "SELECT COALESCE(SUM(`mailbox`.`quota`), 0) FROM `mailbox` WHERE `mailbox`.`domain` = `d`.`domain` AND (`mailbox`.`kind` = '' OR `mailbox`.`kind` = NULL)"], + ['db' => 'stats', 'dt' => 6, 'dummy' => true, 'order_subquery' => "SELECT SUM(bytes) FROM `quota2` WHERE `quota2`.`username` IN (SELECT `username` FROM `mailbox` WHERE `domain` = `d`.`domain`)"], + ['db' => 'defquota', 'dt' => 7], + ['db' => 'maxquota', 'dt' => 8], + ['db' => 'backupmx', 'dt' => 10], + ['db' => 'active', 'dt' => 15], ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/lib/ssp.class.php'; From efcca61f5a4f33bff53758390b3f18a452744cd1 Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Mon, 4 Dec 2023 14:49:07 +0100 Subject: [PATCH 07/32] Mailboxes datatable - server side processing ordering Signed-off-by: Kristian Feldsam --- data/web/js/site/mailbox.js | 8 +------- data/web/json_api.php | 5 +++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index c6f57d457..b1ec94824 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -1002,7 +1002,6 @@ jQuery(function($){ title: lang.last_mail_login, data: 'last_mail_login', searchable: false, - orderable: false, defaultContent: '', responsivePriority: 7, render: function (data, type) { @@ -1016,18 +1015,15 @@ jQuery(function($){ title: lang.last_pw_change, data: 'last_pw_change', searchable: false, - orderable: false, defaultContent: '' }, { title: lang.in_use, data: 'in_use.value', searchable: false, - orderable: false, defaultContent: '', responsivePriority: 9, - className: 'dt-data-w100', - orderData: 24 + className: 'dt-data-w100' }, { title: lang.fname, @@ -1093,7 +1089,6 @@ jQuery(function($){ title: lang.msg_num, data: 'messages', searchable: false, - orderable: false, defaultContent: '', responsivePriority: 5 }, @@ -1119,7 +1114,6 @@ jQuery(function($){ title: lang.active, data: 'active', searchable: false, - orderable: false, defaultContent: '', responsivePriority: 4, render: function (data, type) { diff --git a/data/web/json_api.php b/data/web/json_api.php index 0f0c23b15..403b5a792 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -1057,6 +1057,11 @@ if (isset($_GET['query'])) { $columns = [ ['db' => 'username', 'dt' => 2], ['db' => 'quota', 'dt' => 3], + ['db' => 'last_mail_login', 'dt' => 4, 'dummy' => true, 'order_subquery' => "SELECT MAX(`datetime`) FROM `sasl_log` WHERE `service` != 'SSO' AND `username` = `m`.`username`"], + ['db' => 'last_pw_change', 'dt' => 5, 'dummy' => true, 'order_subquery' => "JSON_EXTRACT(attributes, '$.passwd_update')"], + ['db' => 'in_use', 'dt' => 6, 'dummy' => true, 'order_subquery' => "(SELECT SUM(bytes) FROM `quota2` WHERE `quota2`.`username` = `m`.`username`) / `m`.`quota`"], + ['db' => 'messages', 'dt' => 17, 'dummy' => true, 'order_subquery' => "SELECT SUM(messages) FROM `quota2` WHERE `quota2`.`username` = `m`.`username`"], + ['db' => 'active', 'dt' => 21] ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/lib/ssp.class.php'; From 40fdf99a552eb992aa24f3669ad3d054359db9d6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 20:07:11 +0000 Subject: [PATCH 08/32] Update dependency composer/composer to v2.6.6 Signed-off-by: milkmaker --- data/Dockerfiles/phpfpm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/phpfpm/Dockerfile b/data/Dockerfiles/phpfpm/Dockerfile index 490310336..6717bd582 100644 --- a/data/Dockerfiles/phpfpm/Dockerfile +++ b/data/Dockerfiles/phpfpm/Dockerfile @@ -12,7 +12,7 @@ ARG MEMCACHED_PECL_VERSION=3.2.0 # renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?.*)$ ARG REDIS_PECL_VERSION=6.0.1 # renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?.*)$ -ARG COMPOSER_VERSION=2.6.5 +ARG COMPOSER_VERSION=2.6.6 RUN apk add -U --no-cache autoconf \ aspell-dev \ From efab11720dd144d800ca0bf127c3410516e6bbb3 Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Fri, 22 Dec 2023 10:39:07 +0100 Subject: [PATCH 09/32] add option to skip footer on reply e-mails --- data/conf/rspamd/dynmaps/footer.php | 3 ++- data/conf/rspamd/lua/rspamd.local.lua | 8 ++++++++ data/web/inc/functions.mailbox.inc.php | 6 ++++-- data/web/inc/init_db.inc.php | 3 ++- data/web/lang/lang.de-de.json | 7 ++----- data/web/lang/lang.en-gb.json | 1 + data/web/templates/edit/domain.twig | 8 ++++++++ 7 files changed, 27 insertions(+), 9 deletions(-) diff --git a/data/conf/rspamd/dynmaps/footer.php b/data/conf/rspamd/dynmaps/footer.php index 6e44f5195..36b307c1b 100644 --- a/data/conf/rspamd/dynmaps/footer.php +++ b/data/conf/rspamd/dynmaps/footer.php @@ -49,13 +49,14 @@ $from = $headers['From']; $empty_footer = json_encode(array( 'html' => '', 'plain' => '', + 'skip_replies' => 0, 'vars' => array() )); error_log("FOOTER: checking for domain " . $domain . ", user " . $username . " and address " . $from . PHP_EOL); try { - $stmt = $pdo->prepare("SELECT `plain`, `html`, `mbox_exclude` FROM `domain_wide_footer` + $stmt = $pdo->prepare("SELECT `plain`, `html`, `mbox_exclude`, `skip_replies` FROM `domain_wide_footer` WHERE `domain` = :domain"); $stmt->execute(array( ':domain' => $domain diff --git a/data/conf/rspamd/lua/rspamd.local.lua b/data/conf/rspamd/lua/rspamd.local.lua index 24fa4f8c3..4cc314b0b 100644 --- a/data/conf/rspamd/lua/rspamd.local.lua +++ b/data/conf/rspamd/lua/rspamd.local.lua @@ -567,6 +567,14 @@ rspamd_config:register_symbol({ if footer and type(footer) == "table" and (footer.html and footer.html ~= "" or footer.plain and footer.plain ~= "") then rspamd_logger.infox(rspamd_config, "found domain wide footer for user %s: html=%s, plain=%s, vars=%s", uname, footer.html, footer.plain, footer.vars) + if footer.skip_replies then + in_reply_to = task:get_header_raw('in-reply-to') + if in_reply_to then + rspamd_logger.infox(rspamd_config, "mail is a reply - skip footer") + return + end + end + local envfrom_mime = task:get_from(2) local from_name = "" if envfrom_mime and envfrom_mime[1].name then diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index e965feafb..37e4b83bf 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -3411,6 +3411,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $footers = array(); $footers['html'] = isset($_data['html']) ? $_data['html'] : ''; $footers['plain'] = isset($_data['plain']) ? $_data['plain'] : ''; + $footers['skip_replies'] = isset($_data['skip_replies']) ? (int)$_data['skip_replies'] : 0; $footers['mbox_exclude'] = array(); if (isset($_data["mbox_exclude"])){ if (!is_array($_data["mbox_exclude"])) { @@ -3460,12 +3461,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { try { $stmt = $pdo->prepare("DELETE FROM `domain_wide_footer` WHERE `domain`= :domain"); $stmt->execute(array(':domain' => $domain)); - $stmt = $pdo->prepare("INSERT INTO `domain_wide_footer` (`domain`, `html`, `plain`, `mbox_exclude`) VALUES (:domain, :html, :plain, :mbox_exclude)"); + $stmt = $pdo->prepare("INSERT INTO `domain_wide_footer` (`domain`, `html`, `plain`, `mbox_exclude`, `skip_replies`) VALUES (:domain, :html, :plain, :mbox_exclude, :skip_replies)"); $stmt->execute(array( ':domain' => $domain, ':html' => $footers['html'], ':plain' => $footers['plain'], ':mbox_exclude' => json_encode($footers['mbox_exclude']), + ':skip_replies' => $footers['skip_replies'], )); } catch (PDOException $e) { @@ -4622,7 +4624,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { } try { - $stmt = $pdo->prepare("SELECT `html`, `plain`, `mbox_exclude` FROM `domain_wide_footer` + $stmt = $pdo->prepare("SELECT `html`, `plain`, `mbox_exclude`, `skip_replies` FROM `domain_wide_footer` WHERE `domain` = :domain"); $stmt->execute(array( ':domain' => $domain diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index 4ea79d9bb..68956d158 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -3,7 +3,7 @@ function init_db_schema() { try { global $pdo; - $db_version = "21112023_1644"; + $db_version = "21122023_1526"; $stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); @@ -273,6 +273,7 @@ function init_db_schema() { "html" => "LONGTEXT", "plain" => "LONGTEXT", "mbox_exclude" => "JSON NOT NULL DEFAULT ('[]')", + "skip_replies" => "TINYINT(1) NOT NULL DEFAULT '0'" ), "keys" => array( "primary" => array( diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index dc7b5dfac..dd1377b4c 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -592,6 +592,7 @@ "domain_footer_html": "HTML footer", "domain_footer_info": "Domain wide footer werden allen ausgehenden E-Mails hinzugefügt, die einer Adresse innerhalb dieser Domain gehört.
Die folgenden Variablen können für den Footer benutzt werden:", "domain_footer_plain": "PLAIN footer", + "domain_footer_skip_replies": "Ignoriere Footer bei Antwort E-Mails", "domain_quota": "Domain Speicherplatz gesamt (MiB)", "domains": "Domains", "dont_check_sender_acl": "Absender für Domain %s u. Alias-Domain nicht prüfen", @@ -680,11 +681,7 @@ "unchanged_if_empty": "Unverändert, wenn leer", "username": "Benutzername", "validate_save": "Validieren und speichern", - "pushover_sound": "Ton", - "domain_footer_info_vars": { - "auth_user": "{= auth_user =} - Angemeldeter Benutzername vom MTA", - "from_user": "{= from_user =} - Von Teil des Benutzers z.B. \"moo@mailcow.tld\" wird \"moo\" zurückgeben." - } + "pushover_sound": "Ton" }, "fido2": { "confirm": "Bestätigen", diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index edde80078..173b258a1 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -600,6 +600,7 @@ "custom": "{= foo =} - If mailbox has the custom attribute \"foo\" with value \"bar\" it returns \"bar\"" }, "domain_footer_plain": "PLAIN footer", + "domain_footer_skip_replies": "Ignore footer on reply e-mails", "domain_quota": "Domain quota", "domains": "Domains", "dont_check_sender_acl": "Disable sender check for domain %s (+ alias domains)", diff --git a/data/web/templates/edit/domain.twig b/data/web/templates/edit/domain.twig index 60e88d091..8a700d06a 100644 --- a/data/web/templates/edit/domain.twig +++ b/data/web/templates/edit/domain.twig @@ -305,6 +305,14 @@ +
+ +
+
+ +
+
+
From 38497b04acb872753f3b5d6f00c7c941d2da8ef5 Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Wed, 27 Dec 2023 14:57:27 +0100 Subject: [PATCH 10/32] [Web] use template for default values in mbox and domain creation --- data/web/inc/functions.mailbox.inc.php | 119 +++++++++++++++---------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index e965feafb..7a21b91b0 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -477,17 +477,25 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { 'msg' => 'access_denied' ); return false; + } + $DOMAIN_DEFAULT_ATTRIBUTES = null; + if ($_data['template']){ + $DOMAIN_DEFAULT_ATTRIBUTES = mailbox('get', 'domain_templates', $_data['template'])['attributes']; } + if (empty($DOMAIN_DEFAULT_ATTRIBUTES)){ + $DOMAIN_DEFAULT_ATTRIBUTES = mailbox('get', 'domain_templates')[0]['attributes']; + } + $domain = idn_to_ascii(strtolower(trim($_data['domain'])), 0, INTL_IDNA_VARIANT_UTS46); $description = $_data['description']; if (empty($description)) $description = $domain; - $tags = (array)$_data['tags']; - $aliases = (int)$_data['aliases']; - $mailboxes = (int)$_data['mailboxes']; - $defquota = (int)$_data['defquota']; - $maxquota = (int)$_data['maxquota']; + $tags = (isset($_data['tags'])) ? (array)$_data['tags'] : $DOMAIN_DEFAULT_ATTRIBUTES['tags']; + $aliases = (isset($_data['aliases'])) ? (int)$_data['aliases'] : $DOMAIN_DEFAULT_ATTRIBUTES['max_num_aliases_for_domain']; + $mailboxes = (isset($_data['mailboxes'])) ? (int)$_data['mailboxes'] : $DOMAIN_DEFAULT_ATTRIBUTES['max_num_mboxes_for_domain']; + $defquota = (isset($_data['defquota'])) ? (int)$_data['defquota'] : $DOMAIN_DEFAULT_ATTRIBUTES['def_quota_for_mbox'] / 1024 ** 2; + $maxquota = (isset($_data['maxquota'])) ? (int)$_data['maxquota'] : $DOMAIN_DEFAULT_ATTRIBUTES['max_quota_for_mbox'] / 1024 ** 2; $restart_sogo = (int)$_data['restart_sogo']; - $quota = (int)$_data['quota']; + $quota = (isset($_data['quota'])) ? (int)$_data['quota'] : $DOMAIN_DEFAULT_ATTRIBUTES['max_quota_for_domain'] / 1024 ** 2; if ($defquota > $maxquota) { $_SESSION['return'][] = array( 'type' => 'danger', @@ -520,11 +528,11 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ); return false; } - $active = intval($_data['active']); - $relay_all_recipients = intval($_data['relay_all_recipients']); - $relay_unknown_only = intval($_data['relay_unknown_only']); - $backupmx = intval($_data['backupmx']); - $gal = intval($_data['gal']); + $active = (isset($_data['active'])) ? intval($_data['active']) : $DOMAIN_DEFAULT_ATTRIBUTES['active']; + $relay_all_recipients = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : $DOMAIN_DEFAULT_ATTRIBUTES['relay_all_recipients']; + $relay_unknown_only = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $DOMAIN_DEFAULT_ATTRIBUTES['relay_unknown_only']; + $backupmx = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $DOMAIN_DEFAULT_ATTRIBUTES['backupmx']; + $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $DOMAIN_DEFAULT_ATTRIBUTES['gal']; if ($relay_all_recipients == 1) { $backupmx = '1'; } @@ -625,9 +633,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ); return false; } - if (!empty(intval($_data['rl_value']))) { + $_data['rl_value'] = (isset($_data['rl_value'])) ? intval($_data['rl_value']) : $DOMAIN_DEFAULT_ATTRIBUTES['rl_value']; + $_data['rl_frame'] = (isset($_data['rl_frame'])) ? intval($_data['rl_frame']) : $DOMAIN_DEFAULT_ATTRIBUTES['rl_frame']; + if (!empty($_data['rl_value']) && !empty($_data['rl_frame'])){ ratelimit('edit', 'domain', array('rl_value' => $_data['rl_value'], 'rl_frame' => $_data['rl_frame'], 'object' => $domain)); } + $_data['key_size'] = (isset($_data['key_size'])) ? intval($_data['key_size']) : $DOMAIN_DEFAULT_ATTRIBUTES['key_size']; + $_data['dkim_selector'] = (isset($_data['dkim_selector'])) ? intval($_data['dkim_selector']) : $DOMAIN_DEFAULT_ATTRIBUTES['dkim_selector']; if (!empty($_data['key_size']) && !empty($_data['dkim_selector'])) { if (!empty($redis->hGet('DKIM_SELECTORS', $domain))) { $_SESSION['return'][] = array( @@ -1006,11 +1018,21 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ); return false; } + if (empty($name)) { + $name = $local_part; + } + $MAILBOX_DEFAULT_ATTRIBUTES = null; + if ($_data['template']){ + $MAILBOX_DEFAULT_ATTRIBUTES = mailbox('get', 'mailbox_templates', $_data['template'])['attributes']; + } + if (empty($MAILBOX_DEFAULT_ATTRIBUTES)){ + $MAILBOX_DEFAULT_ATTRIBUTES = mailbox('get', 'mailbox_templates')[0]['attributes']; + } $password = $_data['password']; $password2 = $_data['password2']; $name = ltrim(rtrim($_data['name'], '>'), '<'); - $tags = $_data['tags']; - $quota_m = intval($_data['quota']); + $tags = (isset($_data['tags'])) ? $_data['tags'] : $MAILBOX_DEFAULT_ATTRIBUTES['tags']; + $quota_m = (isset($_data['quota'])) ? intval($_data['quota']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['quota']) / 1024 ** 2; if ((!isset($_SESSION['acl']['unlimited_quota']) || $_SESSION['acl']['unlimited_quota'] != "1") && $quota_m === 0) { $_SESSION['return'][] = array( 'type' => 'danger', @@ -1019,9 +1041,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ); return false; } - if (empty($name)) { - $name = $local_part; - } + if (isset($_data['protocol_access'])) { $_data['protocol_access'] = (array)$_data['protocol_access']; $_data['imap_access'] = (in_array('imap', $_data['protocol_access'])) ? 1 : 0; @@ -1029,7 +1049,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $_data['smtp_access'] = (in_array('smtp', $_data['protocol_access'])) ? 1 : 0; $_data['sieve_access'] = (in_array('sieve', $_data['protocol_access'])) ? 1 : 0; } - $active = intval($_data['active']); + $active = (isset($_data['active'])) ? intval($_data['active']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['active']); $force_pw_update = (isset($_data['force_pw_update'])) ? intval($_data['force_pw_update']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update']); $tls_enforce_in = (isset($_data['tls_enforce_in'])) ? intval($_data['tls_enforce_in']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_in']); $tls_enforce_out = (isset($_data['tls_enforce_out'])) ? intval($_data['tls_enforce_out']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_out']); @@ -1227,12 +1247,29 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $_data['quarantine_notification'] = (in_array('quarantine_notification', $_data['acl'])) ? 1 : 0; $_data['quarantine_category'] = (in_array('quarantine_category', $_data['acl'])) ? 1 : 0; $_data['app_passwds'] = (in_array('app_passwds', $_data['acl'])) ? 1 : 0; + } else { + $_data['spam_alias'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_spam_alias']); + $_data['tls_policy'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_tls_policy']); + $_data['spam_score'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_spam_score']); + $_data['spam_policy'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_spam_policy']); + $_data['delimiter_action'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_delimiter_action']); + $_data['syncjobs'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_syncjobs']); + $_data['eas_reset'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_eas_reset']); + $_data['sogo_profile_reset'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_sogo_profile_reset']); + $_data['pushover'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_pushover']); + $_data['quarantine'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_quarantine']); + $_data['quarantine_attachments'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_quarantine_attachments']); + $_data['quarantine_notification'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_quarantine_notification']); + $_data['quarantine_category'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_quarantine_category']); + $_data['app_passwds'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['acl_app_passwds']); + } + try { $stmt = $pdo->prepare("INSERT INTO `user_acl` (`username`, `spam_alias`, `tls_policy`, `spam_score`, `spam_policy`, `delimiter_action`, `syncjobs`, `eas_reset`, `sogo_profile_reset`, - `pushover`, `quarantine`, `quarantine_attachments`, `quarantine_notification`, `quarantine_category`, `app_passwds`) + `pushover`, `quarantine`, `quarantine_attachments`, `quarantine_notification`, `quarantine_category`, `app_passwds`) VALUES (:username, :spam_alias, :tls_policy, :spam_score, :spam_policy, :delimiter_action, :syncjobs, :eas_reset, :sogo_profile_reset, - :pushover, :quarantine, :quarantine_attachments, :quarantine_notification, :quarantine_category, :app_passwds) "); + :pushover, :quarantine, :quarantine_attachments, :quarantine_notification, :quarantine_category, :app_passwds) "); $stmt->execute(array( ':username' => $username, ':spam_alias' => $_data['spam_alias'], @@ -1251,31 +1288,17 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ':app_passwds' => $_data['app_passwds'] )); } - else { - $stmt = $pdo->prepare("INSERT INTO `user_acl` - (`username`, `spam_alias`, `tls_policy`, `spam_score`, `spam_policy`, `delimiter_action`, `syncjobs`, `eas_reset`, `sogo_profile_reset`, - `pushover`, `quarantine`, `quarantine_attachments`, `quarantine_notification`, `quarantine_category`, `app_passwds`) - VALUES (:username, :spam_alias, :tls_policy, :spam_score, :spam_policy, :delimiter_action, :syncjobs, :eas_reset, :sogo_profile_reset, - :pushover, :quarantine, :quarantine_attachments, :quarantine_notification, :quarantine_category, :app_passwds) "); - $stmt->execute(array( - ':username' => $username, - ':spam_alias' => 0, - ':tls_policy' => 0, - ':spam_score' => 0, - ':spam_policy' => 0, - ':delimiter_action' => 0, - ':syncjobs' => 0, - ':eas_reset' => 0, - ':sogo_profile_reset' => 0, - ':pushover' => 0, - ':quarantine' => 0, - ':quarantine_attachments' => 0, - ':quarantine_notification' => 0, - ':quarantine_category' => 0, - ':app_passwds' => 0 - )); + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), + 'msg' => $e->getMessage() + ); + return false; } + $_data['rl_frame'] = (isset($_data['rl_frame'])) ? $_data['rl_frame'] : $MAILBOX_DEFAULT_ATTRIBUTES['rl_frame']; + $_data['rl_value'] = (isset($_data['rl_value'])) ? $_data['rl_value'] : $MAILBOX_DEFAULT_ATTRIBUTES['rl_value']; if (isset($_data['rl_frame']) && isset($_data['rl_value'])){ ratelimit('edit', 'mailbox', array( 'object' => $username, @@ -1524,17 +1547,17 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $attr["tls_enforce_out"] = isset($_data['tls_enforce_out']) ? intval($_data['tls_enforce_out']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_out']); if (isset($_data['protocol_access'])) { $_data['protocol_access'] = (array)$_data['protocol_access']; - $attr['imap_access'] = (in_array('imap', $_data['protocol_access'])) ? 1 : intval($MAILBOX_DEFAULT_ATTRIBUTES['imap_access']); - $attr['pop3_access'] = (in_array('pop3', $_data['protocol_access'])) ? 1 : intval($MAILBOX_DEFAULT_ATTRIBUTES['pop3_access']); - $attr['smtp_access'] = (in_array('smtp', $_data['protocol_access'])) ? 1 : intval($MAILBOX_DEFAULT_ATTRIBUTES['smtp_access']); - $attr['sieve_access'] = (in_array('sieve', $_data['protocol_access'])) ? 1 : intval($MAILBOX_DEFAULT_ATTRIBUTES['sieve_access']); + $attr['imap_access'] = (in_array('imap', $_data['protocol_access'])) ? 1 : 0; + $attr['pop3_access'] = (in_array('pop3', $_data['protocol_access'])) ? 1 : 0; + $attr['smtp_access'] = (in_array('smtp', $_data['protocol_access'])) ? 1 : 0; + $attr['sieve_access'] = (in_array('sieve', $_data['protocol_access'])) ? 1 : 0; } else { $attr['imap_access'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['imap_access']); $attr['pop3_access'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['pop3_access']); $attr['smtp_access'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['smtp_access']); $attr['sieve_access'] = intval($MAILBOX_DEFAULT_ATTRIBUTES['sieve_access']); - } + } if (isset($_data['acl'])) { $_data['acl'] = (array)$_data['acl']; $attr['acl_spam_alias'] = (in_array('spam_alias', $_data['acl'])) ? 1 : 0; From c24543fea07058100494eaab2cd8c029af16657c Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Wed, 27 Dec 2023 17:32:26 +0100 Subject: [PATCH 11/32] [Web] Fixed form fields bg color in dark mode Signed-off-by: Kristian Feldsam --- data/web/css/themes/mailcow-darkmode.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/web/css/themes/mailcow-darkmode.css b/data/web/css/themes/mailcow-darkmode.css index ea95df970..83befeaf5 100644 --- a/data/web/css/themes/mailcow-darkmode.css +++ b/data/web/css/themes/mailcow-darkmode.css @@ -175,6 +175,9 @@ pre { background-color: #282828; border: 1px solid #555; } +.form-control { + background-color: transparent; +} input.form-control, textarea.form-control { color: #e2e2e2 !important; background-color: #424242 !important; From c61938db23c5bbc76be8172e22a25e89253655e9 Mon Sep 17 00:00:00 2001 From: Dmitriy Alekseev <1865999+dragoangel@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:59:16 +0200 Subject: [PATCH 12/32] [Postfix] Remove pipeling from ehlo keywords as we block it in data restrictions --- data/conf/postfix/main.cf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 237b42635..881c7d797 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -160,7 +160,8 @@ transport_maps = pcre:/opt/postfix/conf/custom_transport.pcre, proxy:mysql:/opt/postfix/conf/sql/mysql_relay_ne.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf smtp_sasl_auth_soft_bounce = no -postscreen_discard_ehlo_keywords = silent-discard, dsn +postscreen_discard_ehlo_keywords = silent-discard, dsn, pipelining, chunking +smtpd_discard_ehlo_keywords = pipelining, chunking compatibility_level = 2 smtputf8_enable = no # Define protocols for SMTPS and submission service From b4bb11320f88070421aff4fafed1a0e784325892 Mon Sep 17 00:00:00 2001 From: Dmitriy Alekseev <1865999+dragoangel@users.noreply.github.com> Date: Fri, 29 Dec 2023 16:04:52 +0200 Subject: [PATCH 13/32] Update main.cf --- data/conf/postfix/main.cf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 881c7d797..7e0a30d92 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -160,8 +160,8 @@ transport_maps = pcre:/opt/postfix/conf/custom_transport.pcre, proxy:mysql:/opt/postfix/conf/sql/mysql_relay_ne.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf smtp_sasl_auth_soft_bounce = no -postscreen_discard_ehlo_keywords = silent-discard, dsn, pipelining, chunking -smtpd_discard_ehlo_keywords = pipelining, chunking +postscreen_discard_ehlo_keywords = silent-discard, dsn, chunking +smtpd_discard_ehlo_keywords = chunking compatibility_level = 2 smtputf8_enable = no # Define protocols for SMTPS and submission service From 68036eeccf4cd2a5ad979c5ec728bfc4296868fa Mon Sep 17 00:00:00 2001 From: Dmitriy Alekseev <1865999+dragoangel@users.noreply.github.com> Date: Fri, 29 Dec 2023 22:06:18 +0200 Subject: [PATCH 14/32] Update main.cf --- data/conf/postfix/main.cf | 1 + 1 file changed, 1 insertion(+) diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 7e0a30d92..a2c2950f2 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -84,6 +84,7 @@ smtp_tls_security_level = dane smtpd_data_restrictions = reject_unauth_pipelining, permit smtpd_delay_reject = yes smtpd_error_sleep_time = 10s +smtpd_forbid_bare_newline = yes smtpd_hard_error_limit = ${stress?1}${stress:5} smtpd_helo_required = yes smtpd_proxy_timeout = 600s From a249e2028dd5ca345aac187d77bcda8d0a1737a7 Mon Sep 17 00:00:00 2001 From: Mathilde <41020854+mthld@users.noreply.github.com> Date: Sat, 30 Dec 2023 10:16:25 +0100 Subject: [PATCH 15/32] Add new SOGoMailHideInlineAttachments option to sogo.conf SOGoMailHideInlineAttachments = YES; will allow to hide inline (body and footer) images being shown as attachments. --- data/conf/sogo/sogo.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/data/conf/sogo/sogo.conf b/data/conf/sogo/sogo.conf index b424efd81..8d4dd93de 100644 --- a/data/conf/sogo/sogo.conf +++ b/data/conf/sogo/sogo.conf @@ -12,6 +12,7 @@ SOGoJunkFolderName= "Junk"; SOGoMailDomain = "sogo.local"; SOGoEnableEMailAlarms = YES; + SOGoMailHideInlineAttachments = YES; SOGoFoldersSendEMailNotifications = YES; SOGoForwardEnabled = YES; From de00c424f49623be282cc5e796e9da027a19030e Mon Sep 17 00:00:00 2001 From: milkmaker Date: Mon, 1 Jan 2024 00:15:22 +0000 Subject: [PATCH 16/32] update postscreen_access.cidr --- data/conf/postfix/postscreen_access.cidr | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/data/conf/postfix/postscreen_access.cidr b/data/conf/postfix/postscreen_access.cidr index 3c51e8362..0497e64a0 100644 --- a/data/conf/postfix/postscreen_access.cidr +++ b/data/conf/postfix/postscreen_access.cidr @@ -1,6 +1,6 @@ -# Whitelist generated by Postwhite v3.4 on Fri Dec 1 00:15:18 UTC 2023 +# Whitelist generated by Postwhite v3.4 on Mon Jan 1 00:15:22 UTC 2024 # https://github.com/stevejenkins/postwhite/ -# 2038 total rules +# 2052 total rules 2a00:1450:4000::/36 permit 2a01:111:f400::/48 permit 2a01:111:f403:8000::/50 permit @@ -13,7 +13,7 @@ 3.70.123.177 permit 3.93.157.0/24 permit 3.129.120.190 permit -3.137.78.75 permit +3.137.16.58 permit 3.210.190.0/24 permit 8.20.114.31 permit 8.25.194.0/23 permit @@ -183,6 +183,8 @@ 50.18.125.237 permit 50.18.126.162 permit 50.31.32.0/19 permit +50.56.130.220 permit +50.56.130.221 permit 51.137.58.21 permit 51.140.75.55 permit 51.144.100.179 permit @@ -596,6 +598,7 @@ 74.208.5.64/26 permit 74.208.122.0/26 permit 74.209.250.0/24 permit +75.2.70.75 permit 76.223.128.0/19 permit 76.223.176.0/20 permit 77.238.176.0/22 permit @@ -1186,6 +1189,7 @@ 98.139.245.208/30 permit 98.139.245.212/31 permit 99.78.197.208/28 permit +99.83.190.102 permit 103.2.140.0/22 permit 103.9.96.0/22 permit 103.28.42.0/24 permit @@ -1460,6 +1464,8 @@ 144.178.38.0/24 permit 145.253.228.160/29 permit 145.253.239.128/29 permit +146.20.14.105 permit +146.20.14.107 permit 146.20.112.0/26 permit 146.20.113.0/24 permit 146.20.191.0/24 permit @@ -1534,6 +1540,10 @@ 163.47.180.0/23 permit 163.114.130.16 permit 163.114.132.120 permit +164.177.132.168 permit +164.177.132.169 permit +164.177.132.170 permit +164.177.132.171 permit 165.173.128.0/24 permit 166.78.68.0/22 permit 166.78.68.221 permit @@ -1726,6 +1736,7 @@ 199.34.22.36 permit 199.59.148.0/22 permit 199.67.80.2 permit +199.67.82.2 permit 199.67.84.0/24 permit 199.67.86.0/24 permit 199.67.88.0/24 permit @@ -1789,6 +1800,7 @@ 204.92.114.187 permit 204.92.114.203 permit 204.92.114.204/31 permit +204.132.224.66 permit 204.141.32.0/23 permit 204.141.42.0/23 permit 204.220.160.0/20 permit @@ -1832,6 +1844,8 @@ 207.67.98.192/27 permit 207.68.176.0/26 permit 207.68.176.96/27 permit +207.97.204.96 permit +207.97.204.97 permit 207.126.144.0/20 permit 207.171.160.0/19 permit 207.211.30.64/26 permit From 6ab45cf6684de9807674ca1c71abd99937e06975 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Mon, 8 Jan 2024 14:43:25 +0100 Subject: [PATCH 17/32] db: bumped version to newer timestamp --- data/web/inc/init_db.inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index 68956d158..6fec0c2f8 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -3,7 +3,7 @@ function init_db_schema() { try { global $pdo; - $db_version = "21122023_1526"; + $db_version = "08012024_1442"; $stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); From b960143045ac5705a7250bffa5daa52838be0d10 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 9 Jan 2024 11:09:35 +0100 Subject: [PATCH 18/32] translation: update de-de.json --- data/web/lang/lang.de-de.json | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index dd1377b4c..3b74d202f 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -588,10 +588,18 @@ "disable_login": "Login verbieten (Mails werden weiterhin angenommen)", "domain": "Domain bearbeiten", "domain_admin": "Domain-Administrator bearbeiten", - "domain_footer": "Domain wide footer", - "domain_footer_html": "HTML footer", - "domain_footer_info": "Domain wide footer werden allen ausgehenden E-Mails hinzugefügt, die einer Adresse innerhalb dieser Domain gehört.
Die folgenden Variablen können für den Footer benutzt werden:", - "domain_footer_plain": "PLAIN footer", + "domain_footer": "Domänenweite Fußzeile", + "domain_footer_html": "Fußzeile im HTML Format", + "domain_footer_info": "Domänenweite Footer (Domain wide footer) werden allen ausgehenden E-Mails hinzugefügt, die einer Adresse innerhalb dieser Domain gehört.
Die folgenden Variablen können für die Fußzeile benutzt werden:", + "domain_footer_info_vars": { + "auth_user": "{= auth_user =} - Angemeldeter Benutzername vom MTA", + "from_user": "{= from_user =} - Absender Teil der E-Mail z.B. für \"moo@mailcow.tld\" wird \"moo\" zurückgeben.", + "from_name": "{= from_name =} - Namen des Absenders z.B. für \"Mailcow <moo@mailcow.tld>\", wird \"Mailcow\" zurückgegeben.", + "from_addr": "{= from_addr =} - Adresse des Absenders.", + "from_domain": "{= from_domain =} - Domain des Absenders", + "custom": "{= foo =} - Wenn die Mailbox das benutzerdefinierte Attribut \"foo\" mit dem Wert \"bar\" hat, wird \"bar\" zurückgegeben." + }, + "domain_footer_plain": "Fußzeile im PLAIN Format", "domain_footer_skip_replies": "Ignoriere Footer bei Antwort E-Mails", "domain_quota": "Domain Speicherplatz gesamt (MiB)", "domains": "Domains", From 5896766fc3e8c732ef7a0822fb3b84a473f4c5e7 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 13 Dec 2023 18:39:21 +0100 Subject: [PATCH 19/32] Update to Alpine 3.19 --- data/Dockerfiles/acme/Dockerfile | 3 ++- data/Dockerfiles/dockerapi/Dockerfile | 3 ++- data/Dockerfiles/netfilter/Dockerfile | 3 ++- data/Dockerfiles/olefy/Dockerfile | 3 ++- data/Dockerfiles/phpfpm/Dockerfile | 2 +- data/Dockerfiles/unbound/Dockerfile | 2 +- data/Dockerfiles/watchdog/Dockerfile | 2 +- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/data/Dockerfiles/acme/Dockerfile b/data/Dockerfiles/acme/Dockerfile index 898dd8b66..27f65e651 100644 --- a/data/Dockerfiles/acme/Dockerfile +++ b/data/Dockerfiles/acme/Dockerfile @@ -1,7 +1,8 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "The Infrastructure Company GmbH " +ARG PIP_BREAK_SYSTEM_PACKAGES=1 RUN apk upgrade --no-cache \ && apk add --update --no-cache \ bash \ diff --git a/data/Dockerfiles/dockerapi/Dockerfile b/data/Dockerfiles/dockerapi/Dockerfile index 3431f9393..23dcbb736 100644 --- a/data/Dockerfiles/dockerapi/Dockerfile +++ b/data/Dockerfiles/dockerapi/Dockerfile @@ -1,7 +1,8 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "The Infrastructure Company GmbH " +ARG PIP_BREAK_SYSTEM_PACKAGES=1 WORKDIR /app RUN apk add --update --no-cache python3 \ diff --git a/data/Dockerfiles/netfilter/Dockerfile b/data/Dockerfiles/netfilter/Dockerfile index 8f76ec639..b6706e307 100644 --- a/data/Dockerfiles/netfilter/Dockerfile +++ b/data/Dockerfiles/netfilter/Dockerfile @@ -1,8 +1,9 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "The Infrastructure Company GmbH " WORKDIR /app +ARG PIP_BREAK_SYSTEM_PACKAGES=1 ENV XTABLES_LIBDIR /usr/lib/xtables ENV PYTHON_IPTABLES_XTABLES_VERSION 12 ENV IPTABLES_LIBDIR /usr/lib diff --git a/data/Dockerfiles/olefy/Dockerfile b/data/Dockerfiles/olefy/Dockerfile index 06d4679f2..bd6e0af35 100644 --- a/data/Dockerfiles/olefy/Dockerfile +++ b/data/Dockerfiles/olefy/Dockerfile @@ -1,6 +1,7 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "The Infrastructure Company GmbH " +ARG PIP_BREAK_SYSTEM_PACKAGES=1 WORKDIR /app #RUN addgroup -S olefy && adduser -S olefy -G olefy \ diff --git a/data/Dockerfiles/phpfpm/Dockerfile b/data/Dockerfiles/phpfpm/Dockerfile index 490310336..c954ac997 100644 --- a/data/Dockerfiles/phpfpm/Dockerfile +++ b/data/Dockerfiles/phpfpm/Dockerfile @@ -1,4 +1,4 @@ -FROM php:8.2-fpm-alpine3.17 +FROM php:8.2-fpm-alpine3.19 LABEL maintainer "The Infrastructure Company GmbH " # renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?.*)$ diff --git a/data/Dockerfiles/unbound/Dockerfile b/data/Dockerfiles/unbound/Dockerfile index b19090835..47dba6d6a 100644 --- a/data/Dockerfiles/unbound/Dockerfile +++ b/data/Dockerfiles/unbound/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "The Infrastructure Company GmbH " diff --git a/data/Dockerfiles/watchdog/Dockerfile b/data/Dockerfiles/watchdog/Dockerfile index 654dea08b..94a28fd0e 100644 --- a/data/Dockerfiles/watchdog/Dockerfile +++ b/data/Dockerfiles/watchdog/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.17 +FROM alpine:3.19 LABEL maintainer "André Peters " # Installation From 333b7ebc0cadfedbf2393eddac4ebc79cae6b187 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Mon, 8 Jan 2024 12:07:22 +0100 Subject: [PATCH 20/32] Fix Alpine 3.19 dependencies --- data/Dockerfiles/dockerapi/Dockerfile | 5 +++-- data/Dockerfiles/dockerapi/main.py | 2 +- data/Dockerfiles/netfilter/Dockerfile | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data/Dockerfiles/dockerapi/Dockerfile b/data/Dockerfiles/dockerapi/Dockerfile index 23dcbb736..d11f5dda6 100644 --- a/data/Dockerfiles/dockerapi/Dockerfile +++ b/data/Dockerfiles/dockerapi/Dockerfile @@ -10,12 +10,13 @@ RUN apk add --update --no-cache python3 \ openssl \ tzdata \ py3-psutil \ + py3-redis \ + py3-async-timeout \ && pip3 install --upgrade pip \ fastapi \ uvicorn \ aiodocker \ - docker \ - aioredis + docker RUN mkdir /app/modules COPY docker-entrypoint.sh /app/ diff --git a/data/Dockerfiles/dockerapi/main.py b/data/Dockerfiles/dockerapi/main.py index f9f02b63f..2842789b5 100644 --- a/data/Dockerfiles/dockerapi/main.py +++ b/data/Dockerfiles/dockerapi/main.py @@ -5,13 +5,13 @@ import json import uuid import async_timeout import asyncio -import aioredis import aiodocker import docker import logging from logging.config import dictConfig from fastapi import FastAPI, Response, Request from modules.DockerApi import DockerApi +from redis import asyncio as aioredis dockerapi = None app = FastAPI() diff --git a/data/Dockerfiles/netfilter/Dockerfile b/data/Dockerfiles/netfilter/Dockerfile index b6706e307..8a561f060 100644 --- a/data/Dockerfiles/netfilter/Dockerfile +++ b/data/Dockerfiles/netfilter/Dockerfile @@ -15,6 +15,7 @@ RUN apk add --virtual .build-deps \ openssl-dev \ && apk add -U python3 \ iptables \ + iptables-dev \ ip6tables \ xtables-addons \ nftables \ From f442378377df798afec9419cb6a9d9be57ed4c42 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 9 Jan 2024 11:18:55 +0100 Subject: [PATCH 21/32] dockerfiles: updated maintainer --- data/Dockerfiles/watchdog/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/watchdog/Dockerfile b/data/Dockerfiles/watchdog/Dockerfile index 94a28fd0e..b94789aac 100644 --- a/data/Dockerfiles/watchdog/Dockerfile +++ b/data/Dockerfiles/watchdog/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.19 -LABEL maintainer "André Peters " +LABEL maintainer "The Infrastructure Company GmbH " # Installation RUN apk add --update \ From 25007b19633a3ae121981206854fb3dc1e9e86a8 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 9 Jan 2024 11:50:22 +0100 Subject: [PATCH 22/32] dockerapi: implemented lifespan function --- data/Dockerfiles/dockerapi/main.py | 97 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/data/Dockerfiles/dockerapi/main.py b/data/Dockerfiles/dockerapi/main.py index 2842789b5..fca61bb02 100644 --- a/data/Dockerfiles/dockerapi/main.py +++ b/data/Dockerfiles/dockerapi/main.py @@ -12,9 +12,56 @@ from logging.config import dictConfig from fastapi import FastAPI, Response, Request from modules.DockerApi import DockerApi from redis import asyncio as aioredis +from contextlib import asynccontextmanager dockerapi = None -app = FastAPI() + +@asynccontextmanager +async def lifespan(app: FastAPI): + global dockerapi + + # Initialize a custom logger + logger = logging.getLogger("dockerapi") + logger.setLevel(logging.INFO) + # Configure the logger to output logs to the terminal + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + formatter = logging.Formatter("%(levelname)s: %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + + logger.info("Init APP") + + # Init redis client + if os.environ['REDIS_SLAVEOF_IP'] != "": + redis_client = redis = await aioredis.from_url(f"redis://{os.environ['REDIS_SLAVEOF_IP']}:{os.environ['REDIS_SLAVEOF_PORT']}/0") + else: + redis_client = redis = await aioredis.from_url("redis://redis-mailcow:6379/0") + + # Init docker clients + sync_docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto') + async_docker_client = aiodocker.Docker(url='unix:///var/run/docker.sock') + + dockerapi = DockerApi(redis_client, sync_docker_client, async_docker_client, logger) + + logger.info("Subscribe to redis channel") + # Subscribe to redis channel + dockerapi.pubsub = redis.pubsub() + await dockerapi.pubsub.subscribe("MC_CHANNEL") + asyncio.create_task(handle_pubsub_messages(dockerapi.pubsub)) + + + yield + + # Close docker connections + dockerapi.sync_docker_client.close() + await dockerapi.async_docker_client.close() + + # Close redis + await dockerapi.pubsub.unsubscribe("MC_CHANNEL") + await dockerapi.redis_client.close() + +app = FastAPI(lifespan=lifespan) # Define Routes @app.get("/host/stats") @@ -144,53 +191,7 @@ async def post_container_update_stats(container_id : str): stats = json.loads(await dockerapi.redis_client.get(container_id + '_stats')) return Response(content=json.dumps(stats, indent=4), media_type="application/json") - -# Events -@app.on_event("startup") -async def startup_event(): - global dockerapi - - # Initialize a custom logger - logger = logging.getLogger("dockerapi") - logger.setLevel(logging.INFO) - # Configure the logger to output logs to the terminal - handler = logging.StreamHandler() - handler.setLevel(logging.INFO) - formatter = logging.Formatter("%(levelname)s: %(message)s") - handler.setFormatter(formatter) - logger.addHandler(handler) - - logger.info("Init APP") - - # Init redis client - if os.environ['REDIS_SLAVEOF_IP'] != "": - redis_client = redis = await aioredis.from_url(f"redis://{os.environ['REDIS_SLAVEOF_IP']}:{os.environ['REDIS_SLAVEOF_PORT']}/0") - else: - redis_client = redis = await aioredis.from_url("redis://redis-mailcow:6379/0") - - # Init docker clients - sync_docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto') - async_docker_client = aiodocker.Docker(url='unix:///var/run/docker.sock') - - dockerapi = DockerApi(redis_client, sync_docker_client, async_docker_client, logger) - - logger.info("Subscribe to redis channel") - # Subscribe to redis channel - dockerapi.pubsub = redis.pubsub() - await dockerapi.pubsub.subscribe("MC_CHANNEL") - asyncio.create_task(handle_pubsub_messages(dockerapi.pubsub)) - -@app.on_event("shutdown") -async def shutdown_event(): - global dockerapi - - # Close docker connections - dockerapi.sync_docker_client.close() - await dockerapi.async_docker_client.close() - - # Close redis - await dockerapi.pubsub.unsubscribe("MC_CHANNEL") - await dockerapi.redis_client.close() + # PubSub Handler async def handle_pubsub_messages(channel: aioredis.client.PubSub): From 1ee3bb42f359fc606a5946325e04769a1f2a71bb Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 9 Jan 2024 11:55:32 +0100 Subject: [PATCH 23/32] compose: updated image tags --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2521e8164..267d1f133 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.1' services: unbound-mailcow: - image: mailcow/unbound:1.18 + image: mailcow/unbound:1.19 environment: - TZ=${TZ} volumes: @@ -107,7 +107,7 @@ services: - rspamd php-fpm-mailcow: - image: mailcow/phpfpm:1.85 + image: mailcow/phpfpm:1.86 command: "php-fpm -d date.timezone=${TZ} -d expose_php=0" depends_on: - redis-mailcow @@ -398,7 +398,7 @@ services: condition: service_started unbound-mailcow: condition: service_healthy - image: mailcow/acme:1.85 + image: mailcow/acme:1.86 dns: - ${IPV4_NETWORK:-172.22.1}.254 environment: @@ -434,7 +434,7 @@ services: - acme netfilter-mailcow: - image: mailcow/netfilter:1.54 + image: mailcow/netfilter:1.55 stop_grace_period: 30s depends_on: - dovecot-mailcow @@ -457,7 +457,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:2.00 + image: mailcow/watchdog:2.01 dns: - ${IPV4_NETWORK:-172.22.1}.254 tmpfs: @@ -529,7 +529,7 @@ services: - watchdog dockerapi-mailcow: - image: mailcow/dockerapi:2.06 + image: mailcow/dockerapi:2.07 security_opt: - label=disable restart: always @@ -564,7 +564,7 @@ services: - solr olefy-mailcow: - image: mailcow/olefy:1.11 + image: mailcow/olefy:1.12 restart: always environment: - TZ=${TZ} From b5a1a18b041cea4b725ea2850721ac9581416971 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Tue, 9 Jan 2024 12:20:30 +0100 Subject: [PATCH 24/32] lang: fixed totp langs --- data/web/lang/lang.de-de.json | 1 + data/web/lang/lang.en-gb.json | 1 + data/web/templates/modals/footer.twig | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index 3b74d202f..3efd5afa0 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -1093,6 +1093,7 @@ "verified_yotp_login": "Yubico-OTP-Anmeldung verifiziert" }, "tfa": { + "authenticators": "Authentikatoren", "api_register": "%s verwendet die Yubico-Cloud-API. Ein API-Key für den Yubico-Stick kann hier bezogen werden.", "confirm": "Bestätigen", "confirm_totp_token": "Bitte bestätigen Sie die Änderung durch Eingabe eines generierten Tokens", diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index 173b258a1..58ad66655 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -1100,6 +1100,7 @@ "verified_yotp_login": "Verified Yubico OTP login" }, "tfa": { + "authenticators": "Authenticators", "api_register": "%s uses the Yubico Cloud API. Please get an API key for your key here", "confirm": "Confirm", "confirm_totp_token": "Please confirm your changes by entering the generated token", diff --git a/data/web/templates/modals/footer.twig b/data/web/templates/modals/footer.twig index ab5630d19..8ff112d5d 100644 --- a/data/web/templates/modals/footer.twig +++ b/data/web/templates/modals/footer.twig @@ -155,7 +155,7 @@ {% if pending_tfa_authmechs["totp"] is defined and pending_tfa_authmechs["u2f"] is not defined %}
  • - Time based OTP + Time-based OTP
    • {% endif %} @@ -173,7 +173,7 @@
    - Authenticators + {{ lang.tfa.authenticators }}
    @@ -216,7 +216,7 @@ - Authenticate + {{ lang.tfa.authenticators }}
    @@ -244,7 +244,7 @@ - Authenticators + {{ lang.tfa.authenticators }}
    From cf9f02adbbec76ae39457952dfb88ec5acce5129 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Wed, 10 Jan 2024 14:43:59 +0100 Subject: [PATCH 25/32] ui: fix alignment secondary --- data/web/css/build/014-mailcow.css | 2 +- data/web/templates/base.twig | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/data/web/css/build/014-mailcow.css b/data/web/css/build/014-mailcow.css index edc6b3d78..6c70a2a55 100644 --- a/data/web/css/build/014-mailcow.css +++ b/data/web/css/build/014-mailcow.css @@ -228,8 +228,8 @@ legend { margin-top: 20px; } .slave-info { - padding: 15px 0px 15px 15px; font-weight: bold; + color: orange; } .alert-hr { margin:3px 0px; diff --git a/data/web/templates/base.twig b/data/web/templates/base.twig index 0b1c60a29..ca744d2a3 100644 --- a/data/web/templates/base.twig +++ b/data/web/templates/base.twig @@ -114,7 +114,9 @@
  • {{ mailcow_cc_username }} ({{ dual_login.username }})
    • {% endif %} {% if not is_master %} -
  • [ slave ]
    • +
    +
  • [ slave ]
    • +
    {% endif %}
    From b29dc3799114089471253d31295a290c239b8488 Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Mon, 15 Jan 2024 15:13:37 +0100 Subject: [PATCH 26/32] unbound: rewrote healthcheck to be more detailed unbound: added comments to rewritten healthcheck --- data/Dockerfiles/unbound/Dockerfile | 4 +- data/Dockerfiles/unbound/healthcheck.sh | 91 +++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/data/Dockerfiles/unbound/Dockerfile b/data/Dockerfiles/unbound/Dockerfile index 47dba6d6a..1e0192f87 100644 --- a/data/Dockerfiles/unbound/Dockerfile +++ b/data/Dockerfiles/unbound/Dockerfile @@ -4,6 +4,8 @@ LABEL maintainer "The Infrastructure Company GmbH " RUN apk add --update --no-cache \ curl \ + bind-tools \ + netcat-openbsd \ unbound \ bash \ openssl \ @@ -21,7 +23,7 @@ COPY docker-entrypoint.sh /docker-entrypoint.sh # healthcheck (nslookup) COPY healthcheck.sh /healthcheck.sh RUN chmod +x /healthcheck.sh -HEALTHCHECK --interval=30s --timeout=10s CMD [ "/healthcheck.sh" ] +HEALTHCHECK --interval=5s --timeout=10s CMD [ "/healthcheck.sh" ] ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/data/Dockerfiles/unbound/healthcheck.sh b/data/Dockerfiles/unbound/healthcheck.sh index 8c4508fb1..ea94f63b3 100644 --- a/data/Dockerfiles/unbound/healthcheck.sh +++ b/data/Dockerfiles/unbound/healthcheck.sh @@ -1,12 +1,89 @@ #!/bin/bash -nslookup mailcow.email 127.0.0.1 1> /dev/null +# Declare log function for logfile inside container +function log_to_file() { + echo "$(date +"%Y-%m-%d %H:%M:%S"): $1" > /var/log/healthcheck.log +} -if [ $? == 0 ]; then - echo "DNS resolution is working!" - exit 0 -else - echo "DNS resolution is not working correctly..." - echo "Maybe check your outbound firewall, as it needs to resolve DNS over TCP AND UDP!" +# General Ping function to check general pingability +function check_ping() { + declare -a ipstoping=("1.1.1.1" "8.8.8.8" "9.9.9.9") + + for ip in "${ipstoping[@]}" ; do + ping -q -c 3 -w 5 "$ip" + if [ $? -ne 0 ]; then + log_to_file "Healthcheck: Couldn't ping $ip for 5 seconds... Gave up!" + log_to_file "Please check your internet connection or firewall rules to fix this error, because a simple ping test should always go through from the unbound container!" + return 1 + fi + done + + log_to_file "Healthcheck: Ping Checks WORKING properly!" + return 0 +} + +# General DNS Resolve Check against Unbound Resolver himself +function check_dns() { + declare -a domains=("mailcow.email" "github.com" "hub.docker.com") + + for domain in "${domains[@]}" ; do + for ((i=1; i<=3; i++)); do + dig +short +timeout=2 +tries=1 "$domain" @127.0.0.1 > /dev/null + if [ $? -ne 0 ]; then + log_to_file "Healthcheck: DNS Resolution Failed on $i attempt! Trying again..." + if [ $i -eq 3 ]; then + log_to_file "Healthcheck: DNS Resolution not possible after $i attempts... Gave up!" + log_to_file "Maybe check your outbound firewall, as it needs to resolve DNS over TCP AND UDP!" + return 1 + fi + fi + done + done + + log_to_file "Healthcheck: DNS Resolver WORKING properly!" + return 0 + +} + +# Simple Netcat Check to connect to common webports +function check_netcat() { + declare -a domains=("mailcow.email" "github.com" "hub.docker.com") + declare -a ports=("80" "443") + + for domain in "${domains[@]}" ; do + for port in "${ports[@]}" ; do + nc -z -w 2 $domain $port + if [ $? -ne 0 ]; then + log_to_file "Healthcheck: Could not reach $domain on Port $port... Gave up!" + log_to_file "Please check your internet connection or firewall rules to fix this error." + return 1 + fi + done + done + + log_to_file "Healthcheck: Netcat Checks WORKING properly!" + return 0 + +} + +# run checks, if check is not returning 0 (return value if check is ok), healthcheck will exit with 1 (marked in docker as unhealthy) +check_ping + +if [ $? -ne 0 ]; then exit 1 fi + +check_dns + +if [ $? -ne 0 ]; then + exit 1 +fi + +check_netcat + +if [ $? -ne 0 ]; then + exit 1 +fi + +log_to_file "Healthcheck: ALL CHECKS WERE SUCCESSFUL! Unbound is healthy!" +exit 0 \ No newline at end of file From 7f6f7e0e9ff608618e5b144bcf18d279610aa3ed Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Mon, 15 Jan 2024 16:34:47 +0100 Subject: [PATCH 27/32] [Web] limit logo file upload --- data/web/inc/functions.customize.inc.php | 18 ++++++++++++++++++ data/web/inc/vars.inc.php | 9 +++++++++ data/web/lang/lang.de-de.json | 2 ++ data/web/lang/lang.en-gb.json | 2 ++ 4 files changed, 31 insertions(+) diff --git a/data/web/inc/functions.customize.inc.php b/data/web/inc/functions.customize.inc.php index 0da8c3563..b72923573 100644 --- a/data/web/inc/functions.customize.inc.php +++ b/data/web/inc/functions.customize.inc.php @@ -2,6 +2,7 @@ function customize($_action, $_item, $_data = null) { global $redis; global $lang; + global $LOGO_LIMITS; switch ($_action) { case 'add': @@ -35,6 +36,23 @@ function customize($_action, $_item, $_data = null) { ); return false; } + if ($_data[$_item]['size'] > $LOGO_LIMITS['max_size']) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_item, $_data), + 'msg' => 'img_size_exceeded' + ); + return false; + } + list($width, $height) = getimagesize($_data[$_item]['tmp_name']); + if ($width > $LOGO_LIMITS['max_width'] || $height > $LOGO_LIMITS['max_height']) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_item, $_data), + 'msg' => 'img_dimensions_exceeded' + ); + return false; + } $image = new Imagick($_data[$_item]['tmp_name']); if ($image->valid() !== true) { $_SESSION['return'][] = array( diff --git a/data/web/inc/vars.inc.php b/data/web/inc/vars.inc.php index 5578dfd3c..afc801e44 100644 --- a/data/web/inc/vars.inc.php +++ b/data/web/inc/vars.inc.php @@ -126,6 +126,15 @@ $MAILCOW_APPS = array( ) ); +// Logo max file size in bytes +$LOGO_LIMITS['max_size'] = 15 * 1024 * 1024; // 15MB + +// Logo max width in pixels +$LOGO_LIMITS['max_width'] = 1920; + +// Logo max height in pixels +$LOGO_LIMITS['max_height'] = 1920; + // Rows until pagination begins $PAGINATION_SIZE = 25; diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index 3efd5afa0..ddadfac63 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -394,7 +394,9 @@ "goto_invalid": "Ziel-Adresse %s ist ungültig", "ham_learn_error": "Ham Lernfehler: %s", "imagick_exception": "Fataler Bildverarbeitungsfehler", + "img_dimensions_exceeded": "Grafik überschreitet die maximale Bildgröße", "img_invalid": "Grafik konnte nicht validiert werden", + "img_size_exceeded": "Grafik überschreitet die maximale Dateigröße", "img_tmp_missing": "Grafik konnte nicht validiert werden: Erstellung temporärer Datei fehlgeschlagen.", "invalid_bcc_map_type": "Ungültiger BCC-Map-Typ", "invalid_destination": "Ziel-Format \"%s\" ist ungültig", diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index 58ad66655..ec97d0aef 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -394,7 +394,9 @@ "goto_invalid": "Goto address %s is invalid", "ham_learn_error": "Ham learn error: %s", "imagick_exception": "Error: Imagick exception while reading image", + "img_dimensions_exceeded": "Image exceeds the maximum image size", "img_invalid": "Cannot validate image file", + "img_size_exceeded": "Image exceeds the maximum file size", "img_tmp_missing": "Cannot validate image file: Temporary file not found", "invalid_bcc_map_type": "Invalid BCC map type", "invalid_destination": "Destination format \"%s\" is invalid", From ac4f131fa8f1629b1c831478d020f5f1b6be4261 Mon Sep 17 00:00:00 2001 From: Kristian Feldsam Date: Wed, 27 Dec 2023 16:29:25 +0100 Subject: [PATCH 28/32] Domains and Mailboxes datatable - server side processing - filtering by tags Signed-off-by: Kristian Feldsam --- data/web/inc/lib/ssp.class.php | 44 ++++++++++++++++++++++++---------- data/web/js/site/mailbox.js | 3 ++- data/web/json_api.php | 2 ++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/data/web/inc/lib/ssp.class.php b/data/web/inc/lib/ssp.class.php index a06e7d09e..503f9b295 100644 --- a/data/web/inc/lib/ssp.class.php +++ b/data/web/inc/lib/ssp.class.php @@ -177,6 +177,7 @@ class SSP { { $globalSearch = array(); $columnSearch = array(); + $joins = array(); $dtColumns = self::pluck( $columns, 'dt' ); if ( isset($request['search']) && $request['search']['value'] != '' ) { @@ -184,13 +185,19 @@ class SSP { for ( $i=0, $ien=count($request['columns']) ; $i<$ien ; $i++ ) { $requestColumn = $request['columns'][$i]; - $columnIdx = array_search( $requestColumn['data'], $dtColumns ); + $columnIdx = array_search( $i, $dtColumns ); $column = $columns[ $columnIdx ]; if ( $requestColumn['searchable'] == 'true' ) { if(!empty($column['db'])){ - $binding = self::bind( $bindings, '%'.$str.'%', PDO::PARAM_STR ); - $globalSearch[] = "`".$tablesAS."`.`".$column['db']."` LIKE ".$binding; + $binding = self::bind( $bindings, '%'.$str.'%', PDO::PARAM_STR ); + + if(isset($column['search']['join'])) { + $joins[] = $column['search']['join']; + $globalSearch[] = $column['search']['where_column'].' LIKE '.$binding; + } else { + $globalSearch[] = "`".$tablesAS."`.`".$column['db']."` LIKE ".$binding; + } } } } @@ -227,12 +234,17 @@ class SSP { implode(' AND ', $columnSearch) : $where .' AND '. implode(' AND ', $columnSearch); } + + $join = ''; + if( count($joins) ) { + $join = implode(' ', $joins); + } if ( $where !== '' ) { $where = 'WHERE '.$where; } - return $where; + return [$join, $where]; } @@ -270,13 +282,14 @@ class SSP { // Build the SQL query string from the request list($select, $order) = self::order( $tablesAS, $request, $columns ); $limit = self::limit( $request, $columns ); - $where = self::filter( $tablesAS, $request, $columns, $bindings ); + list($join, $where) = self::filter( $tablesAS, $request, $columns, $bindings ); // Main query to actually get the data $data = self::sql_exec( $db, $bindings, "SELECT `$tablesAS`.`".implode("`, `$tablesAS`.`", self::pluck($columns, 'db'))."` $select FROM `$table` AS `$tablesAS` + $join $where $order $limit" @@ -284,15 +297,16 @@ class SSP { // Data set length after filtering $resFilterLength = self::sql_exec( $db, $bindings, - "SELECT COUNT(`{$primaryKey}`) + "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` + $join $where" ); $recordsFiltered = $resFilterLength[0][0]; // Total data set length $resTotalLength = self::sql_exec( $db, - "SELECT COUNT(`{$primaryKey}`) + "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS`" ); $recordsTotal = $resTotalLength[0][0]; @@ -362,7 +376,7 @@ class SSP { // Build the SQL query string from the request list($select, $order) = self::order( $tablesAS, $request, $columns ); $limit = self::limit( $request, $columns ); - $where = self::filter( $tablesAS, $request, $columns, $bindings ); + list($join_filter, $where) = self::filter( $tablesAS, $request, $columns, $bindings ); // whereResult can be a simple string, or an assoc. array with a // condition and bindings @@ -388,7 +402,9 @@ class SSP { $select FROM `$table` AS `$tablesAS` $join + $join_filter $where + GROUP BY `{$tablesAS}`.`{$primaryKey}` $order $limit" ); @@ -398,18 +414,22 @@ class SSP { "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` $join - $where" + $join_filter + $where + GROUP BY `{$tablesAS}`.`{$primaryKey}`" ); - $recordsFiltered = $resFilterLength[0][0]; + $recordsFiltered = (isset($resFilterLength[0])) ? $resFilterLength[0][0] : 0; // Total data set length $resTotalLength = self::sql_exec( $db, $bindings, "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` $join - $where" + $join_filter + $where + GROUP BY `{$tablesAS}`.`{$primaryKey}`" ); - $recordsTotal = $resTotalLength[0][0]; + $recordsTotal = (isset($resTotalLength[0])) ? $resTotalLength[0][0] : 0; /* * Output diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index b1ec94824..cc316b713 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -613,7 +613,7 @@ jQuery(function($){ { title: 'Tags', data: 'tags', - searchable: false, + searchable: true, orderable: false, defaultContent: '', className: 'none' @@ -1107,6 +1107,7 @@ jQuery(function($){ { title: 'Tags', data: 'tags', + searchable: true, defaultContent: '', className: 'none' }, diff --git a/data/web/json_api.php b/data/web/json_api.php index 403b5a792..05d549030 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -535,6 +535,7 @@ if (isset($_GET['query'])) { ['db' => 'defquota', 'dt' => 7], ['db' => 'maxquota', 'dt' => 8], ['db' => 'backupmx', 'dt' => 10], + ['db' => 'tags', 'dt' => 14, 'dummy' => true, 'search' => ['join' => 'LEFT JOIN `tags_domain` AS `td` ON `td`.`domain` = `d`.`domain`', 'where_column' => '`td`.`tag_name`']], ['db' => 'active', 'dt' => 15], ]; @@ -1061,6 +1062,7 @@ if (isset($_GET['query'])) { ['db' => 'last_pw_change', 'dt' => 5, 'dummy' => true, 'order_subquery' => "JSON_EXTRACT(attributes, '$.passwd_update')"], ['db' => 'in_use', 'dt' => 6, 'dummy' => true, 'order_subquery' => "(SELECT SUM(bytes) FROM `quota2` WHERE `quota2`.`username` = `m`.`username`) / `m`.`quota`"], ['db' => 'messages', 'dt' => 17, 'dummy' => true, 'order_subquery' => "SELECT SUM(messages) FROM `quota2` WHERE `quota2`.`username` = `m`.`username`"], + ['db' => 'tags', 'dt' => 20, 'dummy' => true, 'search' => ['join' => 'LEFT JOIN `tags_mailbox` AS `tm` ON `tm`.`username` = `m`.`username`', 'where_column' => '`tm`.`tag_name`']], ['db' => 'active', 'dt' => 21] ]; From 99d9a2eacdfffc27604e2b7c572638c456398d19 Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Wed, 17 Jan 2024 09:52:43 +0100 Subject: [PATCH 29/32] [Web] fix mailbox and domain creation --- data/web/inc/functions.mailbox.inc.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index d95ee5349..699709ce9 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -477,12 +477,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { 'msg' => 'access_denied' ); return false; - } + } $DOMAIN_DEFAULT_ATTRIBUTES = null; if ($_data['template']){ $DOMAIN_DEFAULT_ATTRIBUTES = mailbox('get', 'domain_templates', $_data['template'])['attributes']; } - if (empty($DOMAIN_DEFAULT_ATTRIBUTES)){ + if (empty($DOMAIN_DEFAULT_ATTRIBUTES)) { $DOMAIN_DEFAULT_ATTRIBUTES = mailbox('get', 'domain_templates')[0]['attributes']; } @@ -634,12 +634,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { return false; } $_data['rl_value'] = (isset($_data['rl_value'])) ? intval($_data['rl_value']) : $DOMAIN_DEFAULT_ATTRIBUTES['rl_value']; - $_data['rl_frame'] = (isset($_data['rl_frame'])) ? intval($_data['rl_frame']) : $DOMAIN_DEFAULT_ATTRIBUTES['rl_frame']; + $_data['rl_frame'] = (isset($_data['rl_frame'])) ? $_data['rl_frame'] : $DOMAIN_DEFAULT_ATTRIBUTES['rl_frame']; if (!empty($_data['rl_value']) && !empty($_data['rl_frame'])){ ratelimit('edit', 'domain', array('rl_value' => $_data['rl_value'], 'rl_frame' => $_data['rl_frame'], 'object' => $domain)); } $_data['key_size'] = (isset($_data['key_size'])) ? intval($_data['key_size']) : $DOMAIN_DEFAULT_ATTRIBUTES['key_size']; - $_data['dkim_selector'] = (isset($_data['dkim_selector'])) ? intval($_data['dkim_selector']) : $DOMAIN_DEFAULT_ATTRIBUTES['dkim_selector']; + $_data['dkim_selector'] = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : $DOMAIN_DEFAULT_ATTRIBUTES['dkim_selector']; if (!empty($_data['key_size']) && !empty($_data['dkim_selector'])) { if (!empty($redis->hGet('DKIM_SELECTORS', $domain))) { $_SESSION['return'][] = array( @@ -1021,13 +1021,15 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { if (empty($name)) { $name = $local_part; } - $MAILBOX_DEFAULT_ATTRIBUTES = null; + $template_attr = null; if ($_data['template']){ - $MAILBOX_DEFAULT_ATTRIBUTES = mailbox('get', 'mailbox_templates', $_data['template'])['attributes']; + $template_attr = mailbox('get', 'mailbox_templates', $_data['template'])['attributes']; } - if (empty($MAILBOX_DEFAULT_ATTRIBUTES)){ - $MAILBOX_DEFAULT_ATTRIBUTES = mailbox('get', 'mailbox_templates')[0]['attributes']; + if (empty($template_attr)) { + $template_attr = mailbox('get', 'mailbox_templates')[0]['attributes']; } + $MAILBOX_DEFAULT_ATTRIBUTES = array_merge($MAILBOX_DEFAULT_ATTRIBUTES, $template_attr); + $password = $_data['password']; $password2 = $_data['password2']; $name = ltrim(rtrim($_data['name'], '>'), '<'); From accedf028003ad6f0511a41a8773562458038bed Mon Sep 17 00:00:00 2001 From: DerLinkman Date: Wed, 26 Apr 2023 08:37:20 +0000 Subject: [PATCH 30/32] Updated mailcow Components to be ARM64 compatible --- data/Dockerfiles/acme/Dockerfile | 2 +- data/Dockerfiles/clamd/Dockerfile | 10 +- data/Dockerfiles/clamd/clamdcheck.sh | 14 ++ data/Dockerfiles/dockerapi/Dockerfile | 2 +- data/Dockerfiles/dovecot/Dockerfile | 196 +++++++++--------- data/Dockerfiles/dovecot/docker-entrypoint.sh | 4 + data/Dockerfiles/dovecot/quarantine_notify.py | 5 +- data/Dockerfiles/dovecot/quota_notify.py | 2 +- data/Dockerfiles/dovecot/supervisord.conf | 4 + .../dovecot/syslog-ng-redis_slave.conf | 10 +- data/Dockerfiles/dovecot/syslog-ng.conf | 10 +- data/Dockerfiles/postfix/Dockerfile | 2 +- data/Dockerfiles/rspamd/Dockerfile | 4 +- data/Dockerfiles/sogo/Dockerfile | 6 +- data/Dockerfiles/unbound/Dockerfile | 2 +- data/Dockerfiles/watchdog/watchdog.sh | 4 +- docker-compose.yml | 17 +- generate_config.sh | 6 +- helper-scripts/_cold-standby.sh | 30 ++- helper-scripts/backup_and_restore.sh | 45 +++- update.sh | 10 +- 21 files changed, 232 insertions(+), 153 deletions(-) create mode 100644 data/Dockerfiles/clamd/clamdcheck.sh diff --git a/data/Dockerfiles/acme/Dockerfile b/data/Dockerfiles/acme/Dockerfile index 27f65e651..08271bddb 100644 --- a/data/Dockerfiles/acme/Dockerfile +++ b/data/Dockerfiles/acme/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.19 -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " ARG PIP_BREAK_SYSTEM_PACKAGES=1 RUN apk upgrade --no-cache \ diff --git a/data/Dockerfiles/clamd/Dockerfile b/data/Dockerfiles/clamd/Dockerfile index 31a332d7a..8e107516b 100644 --- a/data/Dockerfiles/clamd/Dockerfile +++ b/data/Dockerfiles/clamd/Dockerfile @@ -1,12 +1,14 @@ -FROM clamav/clamav:1.0.3_base +FROM alpine:3.19 -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " RUN apk upgrade --no-cache \ && apk add --update --no-cache \ rsync \ + clamav \ bind-tools \ - bash + bash \ + tini # init COPY clamd.sh /clamd.sh @@ -14,7 +16,9 @@ RUN chmod +x /sbin/tini # healthcheck COPY healthcheck.sh /healthcheck.sh +COPY clamdcheck.sh /usr/local/bin RUN chmod +x /healthcheck.sh +RUN chmod +x /usr/local/bin/clamdcheck.sh HEALTHCHECK --start-period=6m CMD "/healthcheck.sh" ENTRYPOINT [] diff --git a/data/Dockerfiles/clamd/clamdcheck.sh b/data/Dockerfiles/clamd/clamdcheck.sh new file mode 100644 index 000000000..7884d48a9 --- /dev/null +++ b/data/Dockerfiles/clamd/clamdcheck.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -eu + +if [ "${CLAMAV_NO_CLAMD:-}" != "false" ]; then + if [ "$(echo "PING" | nc localhost 3310)" != "PONG" ]; then + echo "ERROR: Unable to contact server" + exit 1 + fi + + echo "Clamd is up" +fi + +exit 0 \ No newline at end of file diff --git a/data/Dockerfiles/dockerapi/Dockerfile b/data/Dockerfiles/dockerapi/Dockerfile index d11f5dda6..3dd1d232e 100644 --- a/data/Dockerfiles/dockerapi/Dockerfile +++ b/data/Dockerfiles/dockerapi/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.19 -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " ARG PIP_BREAK_SYSTEM_PACKAGES=1 WORKDIR /app diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index d1413b2ae..9433dd2ea 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -1,119 +1,115 @@ -FROM debian:bullseye-slim -LABEL maintainer "The Infrastructure Company GmbH " +FROM alpine:3.19 +LABEL maintainer "The Infrastructure Company GmbH GmbH " -ARG DEBIAN_FRONTEND=noninteractive -# renovate: datasource=github-tags depName=dovecot/core versioning=semver-coerced extractVersion=(?.*)$ -ARG DOVECOT=2.3.21 -# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=(?.*)$ -ARG GOSU_VERSION=1.17 -ENV LC_ALL C +# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?.*)$ +ARG GOSU_VERSION=1.16 +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 # Add groups and users before installing Dovecot to not break compatibility -RUN groupadd -g 5000 vmail \ - && groupadd -g 401 dovecot \ - && groupadd -g 402 dovenull \ - && groupadd -g 999 sogo \ - && usermod -a -G sogo nobody \ - && useradd -g vmail -u 5000 vmail -d /var/vmail \ - && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \ - && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull \ - && touch /etc/default/locale \ - && apt-get update \ - && apt-get -y --no-install-recommends install \ - build-essential \ - apt-transport-https \ +RUN addgroup -g 5000 vmail \ + && addgroup -g 401 dovecot \ + && addgroup -g 402 dovenull \ + && sed -i "s/999/99/" /etc/group \ + && addgroup -g 999 sogo \ + && addgroup nobody sogo \ + && adduser -D -u 5000 -G vmail -h /var/vmail vmail \ + && adduser -D -G dovecot -u 401 -h /dev/null -s /sbin/nologin dovecot \ + && adduser -D -G dovenull -u 402 -h /dev/null -s /sbin/nologin dovenull \ + && apk add --no-cache --update \ + bash \ + bind-tools \ + findutils \ + envsubst \ ca-certificates \ - cpanminus \ curl \ - dnsutils \ - dirmngr \ - gettext \ - gnupg2 \ jq \ - libauthen-ntlm-perl \ - libcgi-pm-perl \ - libcrypt-openssl-rsa-perl \ - libcrypt-ssleay-perl \ - libdata-uniqid-perl \ - libdbd-mysql-perl \ - libdbi-perl \ - libdigest-hmac-perl \ - libdist-checkconflicts-perl \ - libencode-imaputf7-perl \ - libfile-copy-recursive-perl \ - libfile-tail-perl \ - libhtml-parser-perl \ - libio-compress-perl \ - libio-socket-inet6-perl \ - libio-socket-ssl-perl \ - libio-tee-perl \ - libipc-run-perl \ - libjson-webtoken-perl \ - liblockfile-simple-perl \ - libmail-imapclient-perl \ - libmodule-implementation-perl \ - libmodule-scandeps-perl \ - libnet-ssleay-perl \ - libpackage-stash-perl \ - libpackage-stash-xs-perl \ - libpar-packer-perl \ - libparse-recdescent-perl \ - libproc-processtable-perl \ - libreadonly-perl \ - libregexp-common-perl \ - libssl-dev \ - libsys-meminfo-perl \ - libterm-readkey-perl \ - libtest-deep-perl \ - libtest-fatal-perl \ - libtest-mock-guard-perl \ - libtest-mockobject-perl \ - libtest-nowarnings-perl \ - libtest-pod-perl \ - libtest-requires-perl \ - libtest-simple-perl \ - libtest-warn-perl \ - libtry-tiny-perl \ - libunicode-string-perl \ - liburi-perl \ - libwww-perl \ - lua-sql-mysql \ + lua \ + lua-cjson \ lua-socket \ + lua-sql-mysql \ + lua5.3-sql-mysql \ + icu-data-full \ + mariadb-connector-c \ + gcompat \ mariadb-client \ + perl \ + perl-ntlm \ + perl-cgi \ + perl-crypt-openssl-rsa \ + perl-utils \ + perl-crypt-ssleay \ + perl-data-uniqid \ + perl-dbd-mysql \ + perl-dbi \ + perl-digest-hmac \ + perl-dist-checkconflicts \ + perl-encode-imaputf7 \ + perl-file-copy-recursive \ + perl-file-tail \ + perl-io-socket-inet6 \ + perl-io-gzip \ + perl-io-socket-ssl \ + perl-io-tee \ + perl-ipc-run \ + perl-json-webtoken \ + perl-mail-imapclient \ + perl-module-implementation \ + perl-module-scandeps \ + perl-net-ssleay \ + perl-package-stash \ + perl-package-stash-xs \ + perl-par-packer \ + perl-parse-recdescent \ + perl-lockfile-simple --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/ \ + libproc \ + perl-readonly \ + perl-regexp-common \ + perl-sys-meminfo \ + perl-term-readkey \ + perl-test-deep \ + perl-test-fatal \ + perl-test-mockobject \ + perl-test-mock-guard \ + perl-test-pod \ + perl-test-requires \ + perl-test-simple \ + perl-test-warn \ + perl-try-tiny \ + perl-unicode-string \ + perl-proc-processtable \ + perl-app-cpanminus \ procps \ - python3-pip \ - redis-server \ - supervisor \ + python3 \ + py3-mysqlclient \ + py3-html2text \ + py3-jinja2 \ + py3-redis \ + redis \ syslog-ng \ - syslog-ng-core \ - syslog-ng-mod-redis \ + syslog-ng-redis \ + syslog-ng-json \ + supervisor \ + tzdata \ wget \ - && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ - && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \ - && chmod +x /usr/local/bin/gosu \ - && gosu nobody true \ - && apt-key adv --fetch-keys https://repo.dovecot.org/DOVECOT-REPO-GPG \ - && echo "deb https://repo.dovecot.org/ce-${DOVECOT}/debian/bullseye bullseye main" > /etc/apt/sources.list.d/dovecot.list \ - && apt-get update \ - && apt-get -y --no-install-recommends install \ - dovecot-lua \ - dovecot-managesieved \ - dovecot-sieve \ + dovecot \ + dovecot-dev \ dovecot-lmtpd \ + dovecot-lua \ dovecot-ldap \ dovecot-mysql \ - dovecot-core \ + dovecot-sql \ + dovecot-submissiond \ + dovecot-pigeonhole-plugin \ dovecot-pop3d \ - dovecot-imapd \ - dovecot-solr \ - && pip3 install mysql-connector-python html2text jinja2 redis \ - && apt-get autoremove --purge -y \ - && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /tmp/* /var/tmp/* /root/.cache/ -# imapsync dependencies -RUN cpan Crypt::OpenSSL::PKCS12 + dovecot-fts-solr \ + && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \ + && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \ + && chmod +x /usr/local/bin/gosu \ + && gosu nobody true + +# RUN cpan LockFile::Simple COPY trim_logs.sh /usr/local/bin/trim_logs.sh COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index b2633c276..f1e2e9668 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -432,4 +432,8 @@ done # May be related to something inside Docker, I seriously don't know touch /etc/dovecot/lua/passwd-verify.lua +if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then + cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf +fi + exec "$@" diff --git a/data/Dockerfiles/dovecot/quarantine_notify.py b/data/Dockerfiles/dovecot/quarantine_notify.py index 65f2a0e60..e8d743b31 100755 --- a/data/Dockerfiles/dovecot/quarantine_notify.py +++ b/data/Dockerfiles/dovecot/quarantine_notify.py @@ -3,11 +3,10 @@ import smtplib import os import sys -import mysql.connector +import MySQLdb from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate -import cgi import jinja2 from jinja2 import Template import json @@ -50,7 +49,7 @@ try: def query_mysql(query, headers = True, update = False): while True: try: - cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user=os.environ.get('DBUSER'), passwd=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8mb4", collation="utf8mb4_general_ci") + cnx = MySQLdb.connect(user=os.environ.get('DBUSER'), password=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8mb4", collation="utf8mb4_general_ci") except Exception as ex: print('%s - trying again...' % (ex)) time.sleep(3) diff --git a/data/Dockerfiles/dovecot/quota_notify.py b/data/Dockerfiles/dovecot/quota_notify.py index 2d7361b81..34b3e0ed9 100755 --- a/data/Dockerfiles/dovecot/quota_notify.py +++ b/data/Dockerfiles/dovecot/quota_notify.py @@ -55,7 +55,7 @@ try: msg.attach(text_part) msg.attach(html_part) msg['To'] = username - p = Popen(['/usr/lib/dovecot/dovecot-lda', '-d', username, '-o', '"plugin/quota=maildir:User quota:noenforcing"'], stdout=PIPE, stdin=PIPE, stderr=STDOUT) + p = Popen(['/usr/libexec/dovecot/dovecot-lda', '-d', username, '-o', '"plugin/quota=maildir:User quota:noenforcing"'], stdout=PIPE, stdin=PIPE, stderr=STDOUT) p.communicate(input=bytes(msg.as_string(), 'utf-8')) domain = username.split("@")[-1] diff --git a/data/Dockerfiles/dovecot/supervisord.conf b/data/Dockerfiles/dovecot/supervisord.conf index a76986409..5b0050002 100644 --- a/data/Dockerfiles/dovecot/supervisord.conf +++ b/data/Dockerfiles/dovecot/supervisord.conf @@ -13,6 +13,10 @@ autostart=true [program:dovecot] command=/usr/sbin/dovecot -F +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 autorestart=true [eventlistener:processes] diff --git a/data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf b/data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf index ea2bcfbfa..f7fc20b7e 100644 --- a/data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf +++ b/data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf @@ -1,4 +1,4 @@ -@version: 3.28 +@version: 4.5 @include "scl.conf" options { chain_hostnames(off); @@ -6,11 +6,11 @@ options { use_dns(no); use_fqdn(no); owner("root"); group("adm"); perm(0640); - stats_freq(0); + stats(freq(0)); bad_hostname("^gconfd$"); }; -source s_src { - unix-stream("/dev/log"); +source s_dgram { + unix-dgram("/dev/log"); internal(); }; destination d_stdout { pipe("/dev/stdout"); }; @@ -36,7 +36,7 @@ filter f_replica { not match("Error: sync: Unknown user in remote" value("MESSAGE")); }; log { - source(s_src); + source(s_dgram); filter(f_replica); destination(d_stdout); filter(f_mail); diff --git a/data/Dockerfiles/dovecot/syslog-ng.conf b/data/Dockerfiles/dovecot/syslog-ng.conf index 2ee4f6241..fcc13587d 100644 --- a/data/Dockerfiles/dovecot/syslog-ng.conf +++ b/data/Dockerfiles/dovecot/syslog-ng.conf @@ -1,4 +1,4 @@ -@version: 3.28 +@version: 4.5 @include "scl.conf" options { chain_hostnames(off); @@ -6,11 +6,11 @@ options { use_dns(no); use_fqdn(no); owner("root"); group("adm"); perm(0640); - stats_freq(0); + stats(freq(0)); bad_hostname("^gconfd$"); }; -source s_src { - unix-stream("/dev/log"); +source s_dgram { + unix-dgram("/dev/log"); internal(); }; destination d_stdout { pipe("/dev/stdout"); }; @@ -36,7 +36,7 @@ filter f_replica { not match("Error: sync: Unknown user in remote" value("MESSAGE")); }; log { - source(s_src); + source(s_dgram); filter(f_replica); destination(d_stdout); filter(f_mail); diff --git a/data/Dockerfiles/postfix/Dockerfile b/data/Dockerfiles/postfix/Dockerfile index bda6e07f2..236062d78 100644 --- a/data/Dockerfiles/postfix/Dockerfile +++ b/data/Dockerfiles/postfix/Dockerfile @@ -1,5 +1,5 @@ FROM debian:bullseye-slim -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C diff --git a/data/Dockerfiles/rspamd/Dockerfile b/data/Dockerfiles/rspamd/Dockerfile index 9d022f826..2511a6f2f 100644 --- a/data/Dockerfiles/rspamd/Dockerfile +++ b/data/Dockerfiles/rspamd/Dockerfile @@ -1,5 +1,5 @@ FROM debian:bullseye-slim -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " ARG DEBIAN_FRONTEND=noninteractive ARG CODENAME=bullseye @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \ dnsutils \ netcat \ && apt-key adv --fetch-keys https://rspamd.com/apt-stable/gpg.key \ - && echo "deb [arch=amd64] https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \ + && echo "deb https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \ && apt-get update \ && apt-get --no-install-recommends -y install rspamd redis-tools procps nano \ && rm -rf /var/lib/apt/lists/* \ diff --git a/data/Dockerfiles/sogo/Dockerfile b/data/Dockerfiles/sogo/Dockerfile index cbc5c93a9..54d676b9a 100644 --- a/data/Dockerfiles/sogo/Dockerfile +++ b/data/Dockerfiles/sogo/Dockerfile @@ -1,8 +1,8 @@ FROM debian:bullseye-slim -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " ARG DEBIAN_FRONTEND=noninteractive -ARG SOGO_DEBIAN_REPOSITORY=http://packages.sogo.nu/nightly/5/debian/ +ARG SOGO_DEBIAN_REPOSITORY=http://www.axis.cz/linux/debian # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?.*)$ ARG GOSU_VERSION=1.17 ENV LC_ALL C @@ -32,7 +32,7 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \ && mkdir /usr/share/doc/sogo \ && touch /usr/share/doc/sogo/empty.sh \ && apt-key adv --keyserver keys.openpgp.org --recv-key 74FFC6D72B925A34B5D356BDF8A27B36A6E2EAE9 \ - && echo "deb ${SOGO_DEBIAN_REPOSITORY} bullseye bullseye" > /etc/apt/sources.list.d/sogo.list \ + && echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} bullseye sogo-v5" > /etc/apt/sources.list.d/sogo.list \ && apt-get update && apt-get install -y --no-install-recommends \ sogo \ sogo-activesync \ diff --git a/data/Dockerfiles/unbound/Dockerfile b/data/Dockerfiles/unbound/Dockerfile index 1e0192f87..cd4dfde8e 100644 --- a/data/Dockerfiles/unbound/Dockerfile +++ b/data/Dockerfiles/unbound/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.19 -LABEL maintainer "The Infrastructure Company GmbH " +LABEL maintainer "The Infrastructure Company GmbH GmbH " RUN apk add --update --no-cache \ curl \ diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index b95bf84b5..d43cb38ac 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -716,8 +716,8 @@ rspamd_checks() { From: watchdog@localhost Empty -' | usr/bin/curl --max-time 10 -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan | jq -rc .default.required_score) - if [[ ${SCORE} != "9999" ]]; then +' | usr/bin/curl --max-time 10 -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan | jq -rc .default.required_score | sed 's/\..*//' ) + if [[ ${SCORE} -ne 9999 ]]; then echo "Rspamd settings check failed, score returned: ${SCORE}" 2>> /tmp/rspamd-mailcow 1>&2 err_count=$(( ${err_count} + 1)) else diff --git a/docker-compose.yml b/docker-compose.yml index 267d1f133..26a0cfe13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: - redis clamd-mailcow: - image: mailcow/clamd:1.63 + image: mailcow/clamd:1.64 restart: always depends_on: unbound-mailcow: @@ -77,7 +77,7 @@ services: - clamd rspamd-mailcow: - image: mailcow/rspamd:1.94 + image: mailcow/rspamd:1.95 stop_grace_period: 30s depends_on: - dovecot-mailcow @@ -171,7 +171,7 @@ services: - phpfpm sogo-mailcow: - image: mailcow/sogo:1.120 + image: mailcow/sogo:1.121 environment: - DBNAME=${DBNAME} - DBUSER=${DBUSER} @@ -203,7 +203,7 @@ services: labels: ofelia.enabled: "true" ofelia.job-exec.sogo_sessions.schedule: "@every 1m" - ofelia.job-exec.sogo_sessions.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-tool expire-sessions $${SOGO_EXPIRE_SESSION} || exit 0\"" + ofelia.job-exec.sogo_sessions.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-tool -v expire-sessions $${SOGO_EXPIRE_SESSION} || exit 0\"" ofelia.job-exec.sogo_ealarms.schedule: "@every 1m" ofelia.job-exec.sogo_ealarms.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-ealarms-notify -p /etc/sogo/sieve.creds || exit 0\"" ofelia.job-exec.sogo_eautoreply.schedule: "@every 5m" @@ -218,7 +218,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.26 + image: mailcow/dovecot:1.27 depends_on: - mysql-mailcow dns: @@ -298,7 +298,7 @@ services: - dovecot postfix-mailcow: - image: mailcow/postfix:1.73 + image: mailcow/postfix:1.74 depends_on: mysql-mailcow: condition: service_started @@ -547,8 +547,10 @@ services: aliases: - dockerapi + + ##### Will be removed soon ##### solr-mailcow: - image: mailcow/solr:1.8.1 + image: mailcow/solr:1.8.2 restart: always volumes: - solr-vol-1:/opt/solr/server/solr/dovecot-fts/data @@ -562,6 +564,7 @@ services: mailcow-network: aliases: - solr + ################################ olefy-mailcow: image: mailcow/olefy:1.12 diff --git a/generate_config.sh b/generate_config.sh index 3b2928937..2986f1689 100755 --- a/generate_config.sh +++ b/generate_config.sh @@ -34,7 +34,7 @@ if docker compose > /dev/null 2>&1; then echo -e "\e[33mNotice: You´ll have to update this Compose Version via your Package Manager manually!\e[0m" else echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m" - echo -e "\e[31mPlease update/install it manually regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease update/install it manually regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi elif docker-compose > /dev/null 2>&1; then @@ -47,14 +47,14 @@ elif docker-compose > /dev/null 2>&1; then echo -e "\e[33mNotice: For an automatic update of docker-compose please use the update_compose.sh scripts located at the helper-scripts folder.\e[0m" else echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m" - echo -e "\e[31mPlease update/install manually regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease update/install manually regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi fi else echo -e "\e[31mCannot find Docker Compose.\e[0m" - echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi diff --git a/helper-scripts/_cold-standby.sh b/helper-scripts/_cold-standby.sh index 0e8885a3e..ff0512e07 100755 --- a/helper-scripts/_cold-standby.sh +++ b/helper-scripts/_cold-standby.sh @@ -2,6 +2,7 @@ PATH=${PATH}:/opt/bin DATE=$(date +%Y-%m-%d_%H_%M_%S) +LOCAL_ARCH=$(uname -m) export LC_ALL=C echo @@ -148,6 +149,9 @@ else echo -e "\e[31mCannot find any Docker Compose on remote, exiting...\e[0m" exit 1 fi + + REMOTE_ARCH=$(ssh -o StrictHostKeyChecking=no -i "${REMOTE_SSH_KEY}" ${REMOTE_SSH_HOST} -p ${REMOTE_SSH_PORT} "uname -m") + } SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) @@ -164,6 +168,17 @@ echo -e "\033[1mFound compose project name ${CMPS_PRJ} for ${MAILCOW_HOSTNAME}\0 echo -e "\033[1mFound SQL ${SQLIMAGE}\033[0m" echo +# Print Message if Local Arch and Remote Arch is not the same +if [[ $LOCAL_ARCH != $REMOTE_ARCH ]]; then + echo + echo -e "\e[1;33m!!!!!!!!!!!!!!!!!!!!!!!!!! CAUTION !!!!!!!!!!!!!!!!!!!!!!!!!!\e[0m" + echo -e "\e[3;33mDetected Architecture missmatch from source to destination...\e[0m" + echo -e "\e[3;33mYour backup is transferred but some volumes might be skipped!\e[0m" + echo -e "\e[1;33m!!!!!!!!!!!!!!!!!!!!!!!!!! CAUTION !!!!!!!!!!!!!!!!!!!!!!!!!!\e[0m" + echo + sleep 2 +fi + # Make sure destination exists, rsync can fail under some circumstances echo -e "\033[1mPreparing remote...\033[0m" if ! ssh -o StrictHostKeyChecking=no \ @@ -248,8 +263,21 @@ for vol in $(docker volume ls -qf name="${CMPS_PRJ}"); do # Cleanup rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/" - else + elif [[ "${vol}" =~ "rspamd-vol-1" ]]; then + # Exclude rspamd-vol-1 if the Architectures are not the same on source and destination due to compatibility issues. + if [[ $LOCAL_ARCH == $REMOTE_ARCH ]]; then + echo -e "\033[1mSynchronizing ${vol} from local ${mountpoint}...\033[0m" + rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \ + -i \"${REMOTE_SSH_KEY}\" \ + -p ${REMOTE_SSH_PORT}" \ + "${mountpoint}/" root@${REMOTE_SSH_HOST}:"${mountpoint}" + else + echo -e "\e[1;31mSkipping ${vol} from local maschine due to incompatiblity between different architecture...\e[0m" + sleep 2 + continue + fi + else echo -e "\033[1mSynchronizing ${vol} from local ${mountpoint}...\033[0m" rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \ -i \"${REMOTE_SSH_KEY}\" \ diff --git a/helper-scripts/backup_and_restore.sh b/helper-scripts/backup_and_restore.sh index ee9f0202d..03390927b 100755 --- a/helper-scripts/backup_and_restore.sh +++ b/helper-scripts/backup_and_restore.sh @@ -53,6 +53,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml ENV_FILE=${SCRIPT_DIR}/../.env THREADS=$(echo ${THREADS:-1}) +ARCH=$(uname -m) if ! [[ "${THREADS}" =~ ^[1-9]+$ ]] ; then echo "Thread input is not a number!" @@ -96,6 +97,7 @@ function backup() { mkdir -p "${BACKUP_LOCATION}/mailcow-${DATE}" chmod 755 "${BACKUP_LOCATION}/mailcow-${DATE}" cp "${SCRIPT_DIR}/../mailcow.conf" "${BACKUP_LOCATION}/mailcow-${DATE}" + touch "${BACKUP_LOCATION}/mailcow-${DATE}/.$ARCH" for bin in docker; do if [[ -z $(which ${bin}) ]]; then >&2 echo -e "\e[31mCannot find ${bin} in local PATH, exiting...\e[0m" @@ -231,12 +233,29 @@ function restore() { docker start $(docker ps -aqf name=dovecot-mailcow) ;; rspamd) - docker stop $(docker ps -qf name=rspamd-mailcow) - docker run -it --name mailcow-backup --rm \ - -v ${RESTORE_LOCATION}:/backup:z \ - -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \ - ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz - docker start $(docker ps -aqf name=rspamd-mailcow) + if [[ $(find "${RESTORE_LOCATION}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') == "" ]]; then + echo -e "\e[33mCould not find a architecture signature of the loaded backup... Maybe the backup was done before the multiarch update?" + sleep 2 + echo -e "Continuing anyhow. If rspamd is crashing opon boot try remove the rspamd volume with docker volume rm ${CMPS_PRJ}_rspamd-vol-1 after you've stopped the stack.\e[0m" + sleep 2 + docker stop $(docker ps -qf name=rspamd-mailcow) + docker run -it --name mailcow-backup --rm \ + -v ${RESTORE_LOCATION}:/backup:z \ + -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \ + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz + docker start $(docker ps -aqf name=rspamd-mailcow) + elif [[ $ARCH != $(find "${RESTORE_LOCATION}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') ]]; then + echo -e "\e[31mThe Architecture of the backed up mailcow OS is different then your restoring mailcow OS..." + sleep 2 + echo -e "Skipping rspamd due to compatibility issues!\e[0m" + else + docker stop $(docker ps -qf name=rspamd-mailcow) + docker run -it --name mailcow-backup --rm \ + -v ${RESTORE_LOCATION}:/backup:z \ + -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \ + ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz + docker start $(docker ps -aqf name=rspamd-mailcow) + fi ;; postfix) docker stop $(docker ps -qf name=postfix-mailcow) @@ -360,9 +379,17 @@ elif [[ ${1} == "restore" ]]; then FILE_SELECTION[${i}]="redis" ((i++)) elif [[ ${file} =~ rspamd ]]; then - echo "[ ${i} ] - Rspamd data" - FILE_SELECTION[${i}]="rspamd" - ((i++)) + if [[ $(find "${FOLDER_SELECTION[${input_sel}]}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') == "" ]]; then + echo "[ ${i} ] - Rspamd data (unkown Arch detected, restore with caution!)" + FILE_SELECTION[${i}]="rspamd" + ((i++)) + elif [[ $ARCH != $(find "${FOLDER_SELECTION[${input_sel}]}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') ]]; then + echo -e "\e[31m[ NaN ] - Rspamd data (incompatible Arch, cannot restore it)\e[0m" + else + echo "[ ${i} ] - Rspamd data" + FILE_SELECTION[${i}]="rspamd" + ((i++)) + fi elif [[ ${file} =~ postfix ]]; then echo "[ ${i} ] - Postfix data" FILE_SELECTION[${i}]="postfix" diff --git a/update.sh b/update.sh index 47ae1f8ba..623242d5b 100755 --- a/update.sh +++ b/update.sh @@ -181,7 +181,7 @@ if ! [[ "${DOCKER_COMPOSE_VERSION}" =~ ^(native|standalone)$ ]]; then echo -e "\e[33mNotice: You'll have to update this Compose Version via your Package Manager manually!\e[0m" else echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m" - echo -e "\e[31mPlease update/install it manually regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease update/install it manually regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi elif docker-compose > /dev/null 2>&1; then @@ -196,14 +196,14 @@ if ! [[ "${DOCKER_COMPOSE_VERSION}" =~ ^(native|standalone)$ ]]; then echo -e "\e[33mNotice: For an automatic update of docker-compose please use the update_compose.sh scripts located at the helper-scripts folder.\e[0m" else echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m" - echo -e "\e[31mPlease update/install regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease update/install regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi fi else echo -e "\e[31mCannot find Docker Compose.\e[0m" - echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi @@ -216,7 +216,7 @@ elif [ "${DOCKER_COMPOSE_VERSION}" == "native" ]; then if ! $COMPOSE_COMMAND > /dev/null 2>&1 || ! $COMPOSE_COMMAND --version | grep "^2." > /dev/null 2>&1; then # IF it cannot find Standalone in > 2.X, then script stops echo -e "\e[31mCannot find Docker Compose or the Version is lower then 2.X.X.\e[0m" - echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi # If it finds the standalone Plugin it will use this instead and change the mailcow.conf Variable accordingly @@ -236,7 +236,7 @@ elif [ "${DOCKER_COMPOSE_VERSION}" == "standalone" ]; then if ! $COMPOSE_COMMAND > /dev/null 2>&1; then # IF it cannot find Native in > 2.X, then script stops echo -e "\e[31mCannot find Docker Compose.\e[0m" - echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/i_u_m/i_u_m_install/\e[0m" + echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/install/\e[0m" exit 1 fi # If it finds the native Plugin it will use this instead and change the mailcow.conf Variable accordingly From cc3adbe78c815e07ef68cc7f2e355b7dc9d03d2c Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Wed, 17 Jan 2024 12:04:01 +0100 Subject: [PATCH 31/32] [Web] fix datatables ssp queries --- data/web/inc/lib/ssp.class.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data/web/inc/lib/ssp.class.php b/data/web/inc/lib/ssp.class.php index 503f9b295..ea6109ac3 100644 --- a/data/web/inc/lib/ssp.class.php +++ b/data/web/inc/lib/ssp.class.php @@ -291,13 +291,14 @@ class SSP { FROM `$table` AS `$tablesAS` $join $where + GROUP BY `{$tablesAS}`.`{$primaryKey}` $order $limit" ); // Data set length after filtering $resFilterLength = self::sql_exec( $db, $bindings, - "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) + "SELECT COUNT(DISTINCT `{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` $join $where" @@ -411,12 +412,11 @@ class SSP { // Data set length after filtering $resFilterLength = self::sql_exec( $db, $bindings, - "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) + "SELECT COUNT(DISTINCT `{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` $join $join_filter - $where - GROUP BY `{$tablesAS}`.`{$primaryKey}`" + $where" ); $recordsFiltered = (isset($resFilterLength[0])) ? $resFilterLength[0][0] : 0; @@ -424,10 +424,9 @@ class SSP { $resTotalLength = self::sql_exec( $db, $bindings, "SELECT COUNT(`{$tablesAS}`.`{$primaryKey}`) FROM `$table` AS `$tablesAS` - $join - $join_filter - $where - GROUP BY `{$tablesAS}`.`{$primaryKey}`" + $join + $join_filter + $where" ); $recordsTotal = (isset($resTotalLength[0])) ? $resTotalLength[0][0] : 0; From 90a7cff2c9061b0ffd0cba63ade68e3a313d4b61 Mon Sep 17 00:00:00 2001 From: FreddleSpl0it Date: Wed, 17 Jan 2024 12:05:51 +0100 Subject: [PATCH 32/32] [Rspamd] check if footer.skip_replies is not 0 --- data/conf/rspamd/lua/rspamd.local.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/conf/rspamd/lua/rspamd.local.lua b/data/conf/rspamd/lua/rspamd.local.lua index 4cc314b0b..45506f3b5 100644 --- a/data/conf/rspamd/lua/rspamd.local.lua +++ b/data/conf/rspamd/lua/rspamd.local.lua @@ -567,7 +567,7 @@ rspamd_config:register_symbol({ if footer and type(footer) == "table" and (footer.html and footer.html ~= "" or footer.plain and footer.plain ~= "") then rspamd_logger.infox(rspamd_config, "found domain wide footer for user %s: html=%s, plain=%s, vars=%s", uname, footer.html, footer.plain, footer.vars) - if footer.skip_replies then + if footer.skip_replies ~= 0 then in_reply_to = task:get_header_raw('in-reply-to') if in_reply_to then rspamd_logger.infox(rspamd_config, "mail is a reply - skip footer")