Deliver aggregate SMTP TLS reports

This commit is contained in:
Mario Fetka
2026-07-21 10:09:42 +02:00
parent 632707224f
commit b02dd492eb
19 changed files with 1454 additions and 9 deletions
+1
View File
@@ -85,6 +85,7 @@ endif()
# Secure IMAP, POP3 and SMTP client transport.
find_package(CURL 7.61 REQUIRED)
find_package(ZLIB REQUIRED)
# Native DNS queries, authoritative propagation checks and RFC 2136 updates
# used by ACME DNS-01 providers.
+1
View File
@@ -22,6 +22,7 @@ Build-Depends:
libgnutls28-dev,
libsqlite3-dev,
libcurl4-gnutls-dev,
zlib1g-dev,
libldns-dev,
libmailutils-dev (>= 1:3.19-1+bongo1),
libgsasl-dev (>= 2.2.0),
+2
View File
@@ -146,6 +146,8 @@ inputs and outputs is absent from the ZIP.
| SMTP-23 | Fallback client-address header only for configured trusted peers | | | |
| SMTP-24 | Incoming PROXY v1/v2 only from configured proxy networks | | | |
| SMTP-25 | Malformed commands, long lines, floods, timeout, and disconnect | | | |
| SMTP-28 | TLSRPT preserves the exact DNS policy, aggregates completed UTC days, and emits valid RFC 8460 JSON | | | |
| SMTP-29 | TLSRPT gzip mail/HTTPS delivery, DKIM requirement, random delay, durable retry, expiry, and report-loop suppression | | | |
## IMAP4rev2, IMAP4rev1, and legacy IMAP4
+19
View File
@@ -39,6 +39,25 @@ cached policy remains authoritative through its `max_age` when DNS discovery
or HTTPS refresh temporarily fails. DANE always takes precedence over
MTA-STS.
When outbound TLS reporting is enabled, the SMTP delivery agent records one
final TLS policy result for each attempted destination. The worker aggregates
completed UTC days into RFC 8460 JSON reports without losing late events or
double-counting retries. It preserves the exact DNS TLSRPT policy that was
used for delivery, waits a randomized interval of up to four hours, and sends
one durable delivery to every `mailto:` and `https:` reporting URI. Reports
are gzip-compressed; mail reports use `multipart/report` with an
`application/tlsrpt+gzip` attachment, while HTTPS reports use a validated
HTTPS POST. Temporary failures are retried with persisted exponential backoff
for 24 hours after the initial attempt.
TLS report messages carry `TLS-Required: No` and bypass outbound DANE,
MTA-STS, and TLSRPT accounting so a receiver's TLS failure cannot create a
report loop. They still prefer STARTTLS. RFC 8460 requires email reports to be
DKIM-signed, so `mailto:` delivery remains pending unless outbound DKIM is
enabled and the configured key for the reporting domain is readable. The
submitter defaults to the DKIM signing domain, or otherwise to the configured
Bongo host name. HTTPS report delivery does not depend on DKIM.
The resolver named in `resolv.conf` must therefore be a trusted validating
resolver reached over a trusted local path; Bongo must not trust an AD bit
received from an untrusted network resolver. GnuTLS/libdane also needs its
+47
View File
@@ -96,6 +96,25 @@ typedef struct BongoTlsRptEvent {
typedef struct BongoTlsRptStore BongoTlsRptStore;
typedef struct BongoTlsRptDelivery {
int64_t id;
int endpoint_type;
int attempts;
int64_t day_start;
int64_t created_at;
int64_t expires_at;
char report_domain[BONGO_TLSRPT_DOMAIN_SIZE];
char report_id[BONGO_TLSRPT_DOMAIN_SIZE * 2];
char endpoint_uri[BONGO_TLSRPT_URI_SIZE];
char *json;
} BongoTlsRptDelivery;
typedef enum BongoTlsRptDeliveryResult {
BONGO_TLSRPT_DELIVERY_RETRY = 0,
BONGO_TLSRPT_DELIVERY_SUCCESS = 1,
BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE = 2
} BongoTlsRptDeliveryResult;
/* Parse one complete TXT value after DNS character-string concatenation. */
int BongoTlsRptParseRecord(const char *record, BongoTlsRptDnsPolicy *policy);
@@ -127,6 +146,34 @@ int BongoTlsRptStoreDeleteReport(BongoTlsRptStore *store,
int64_t day_start,
const char *reporting_policy);
/*
* Atomically turn the oldest completed reporting-policy group into one
* durable delivery per RUA endpoint. The corresponding event rows are
* removed in the same SQLite write transaction, so events recorded after
* the snapshot cannot be lost. Returns 1 when a batch was prepared, 0 when
* no complete day is available, and -1 on error.
*/
int BongoTlsRptStorePrepareNext(BongoTlsRptStore *store,
int64_t before_day_start,
const char *organization_name,
const char *contact_info,
const char *submitter,
int64_t now,
int64_t initial_not_before);
/* Returns 1 for a due delivery, 0 when none is due, and -1 on error. */
int BongoTlsRptStoreNextDelivery(BongoTlsRptStore *store, int64_t now,
BongoTlsRptDelivery *delivery,
int64_t *next_not_before);
void BongoTlsRptDeliveryFree(BongoTlsRptDelivery *delivery);
int BongoTlsRptStoreFinishDelivery(BongoTlsRptStore *store,
int64_t delivery_id,
BongoTlsRptDeliveryResult result,
int64_t next_attempt,
const char *error);
int BongoTlsRptStoreCleanupExpired(BongoTlsRptStore *store, int64_t now);
const char *BongoTlsRptPolicyTypeName(BongoTlsRptPolicyType type);
const char *BongoTlsRptResultTypeName(BongoTlsRptResultType type);
+22
View File
@@ -7,6 +7,8 @@ add_executable(bongoworker
jobs/acme-renew.c
jobs/scanner-health.c
jobs/scanner-health-support.c
jobs/tlsrpt-delivery.c
jobs/tlsrpt-delivery-support.c
)
target_include_directories(bongoworker PRIVATE
@@ -26,6 +28,9 @@ target_link_libraries(bongoworker
bongoconnio
bongocal
SQLite3::SQLite3
bongotlsrpt
CURL::libcurl
ZLIB::ZLIB
Threads::Threads
)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
@@ -42,6 +47,8 @@ if(BUILD_TESTING)
jobs/acme-renew.c
jobs/scanner-health.c
jobs/scanner-health-support.c
jobs/tlsrpt-delivery.c
jobs/tlsrpt-delivery-support.c
)
target_include_directories(worker-config-test PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
@@ -58,6 +65,9 @@ if(BUILD_TESTING)
bongomsgapi
bongocal
SQLite3::SQLite3
bongotlsrpt
CURL::libcurl
ZLIB::ZLIB
GnuTLS::GnuTLS
Threads::Threads
)
@@ -77,4 +87,16 @@ if(BUILD_TESTING)
SQLite3::SQLite3
)
add_test(NAME worker-scanner-health COMMAND worker-scanner-health-test)
add_executable(worker-tlsrpt-delivery-test
tests/tlsrpt-delivery-test.c
jobs/tlsrpt-delivery-support.c
)
target_include_directories(worker-tlsrpt-delivery-test PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/jobs
)
target_link_libraries(worker-tlsrpt-delivery-test PRIVATE
ZLIB::ZLIB
)
add_test(NAME worker-tlsrpt-delivery COMMAND worker-tlsrpt-delivery-test)
endif()
+3
View File
@@ -27,10 +27,13 @@ int BongoWorkerRunAcmeRenew(char *error, size_t error_size,
int64_t *not_before);
int BongoWorkerRunScannerHealth(char *error, size_t error_size,
int64_t *not_before);
int BongoWorkerRunTlsRptDelivery(char *error, size_t error_size,
int64_t *not_before);
static const BongoWorkerJob Jobs[] = {
{"acme-renew", BongoWorkerRunAcmeRenew},
{"scanner-health", BongoWorkerRunScannerHealth},
{"tlsrpt-delivery", BongoWorkerRunTlsRptDelivery},
{NULL, NULL}
};
@@ -0,0 +1,347 @@
/****************************************************************************
* <Novell-copyright>
* Copyright (c) 2001 Novell, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 2 of the GNU General Public License.
* </Novell-copyright>
****************************************************************************/
#include "tlsrpt-delivery-support.h"
#include <zlib.h>
#include <ctype.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
typedef struct Buffer {
char *data;
size_t length;
size_t capacity;
} Buffer;
static void
SetError(char *error, size_t error_size, const char *format, ...)
{
va_list arguments;
if (!error || error_size == 0U) return;
va_start(arguments, format);
vsnprintf(error, error_size, format, arguments);
va_end(arguments);
}
static int
Reserve(Buffer *buffer, size_t extra)
{
size_t needed;
size_t capacity;
char *replacement;
if (extra > SIZE_MAX - buffer->length - 1U) return 0;
needed = buffer->length + extra + 1U;
if (needed <= buffer->capacity) return 1;
capacity = buffer->capacity ? buffer->capacity : 1024U;
while (capacity < needed) {
if (capacity > SIZE_MAX / 2U) {
capacity = needed;
break;
}
capacity *= 2U;
}
replacement = realloc(buffer->data, capacity);
if (!replacement) return 0;
buffer->data = replacement;
buffer->capacity = capacity;
return 1;
}
static int
AppendN(Buffer *buffer, const char *data, size_t length)
{
if (!Reserve(buffer, length)) return 0;
memcpy(buffer->data + buffer->length, data, length);
buffer->length += length;
buffer->data[buffer->length] = '\0';
return 1;
}
static int
Append(Buffer *buffer, const char *data)
{
return AppendN(buffer, data, strlen(data));
}
static int
AppendF(Buffer *buffer, const char *format, ...)
{
va_list arguments;
va_list copied;
int length;
va_start(arguments, format);
va_copy(copied, arguments);
length = vsnprintf(NULL, 0, format, copied);
va_end(copied);
if (length < 0 || !Reserve(buffer, (size_t)length)) {
va_end(arguments);
return 0;
}
(void)vsnprintf(buffer->data + buffer->length,
buffer->capacity - buffer->length, format, arguments);
va_end(arguments);
buffer->length += (size_t)length;
return 1;
}
static int
HeaderSafe(const char *value)
{
const unsigned char *cursor = (const unsigned char *)value;
if (!cursor || !*cursor || strlen(value) > 900U) return 0;
for (; *cursor; cursor++)
if (*cursor < 0x21 || *cursor == 0x7f) return 0;
return 1;
}
int
BongoWorkerTlsRptGzip(const char *json, unsigned char **compressed,
size_t *compressed_size,
char *error, size_t error_size)
{
z_stream stream;
size_t input_size;
uLong bound;
int status;
int result = 0;
if (error && error_size) error[0] = '\0';
if (!json || !compressed || !compressed_size) return 0;
*compressed = NULL;
*compressed_size = 0U;
input_size = strlen(json);
if (input_size > UINT_MAX) {
SetError(error, error_size, "TLSRPT JSON is too large to compress");
return 0;
}
memset(&stream, 0, sizeof(stream));
status = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY);
if (status != Z_OK) {
SetError(error, error_size, "cannot initialize gzip compressor");
return 0;
}
bound = deflateBound(&stream, (uLong)input_size);
if (bound > UINT_MAX) {
SetError(error, error_size, "compressed TLSRPT report is too large");
goto done;
}
*compressed = malloc((size_t)bound ? (size_t)bound : 1U);
if (!*compressed) {
SetError(error, error_size, "out of memory compressing TLSRPT report");
goto done;
}
stream.next_in = (Bytef *)(uintptr_t)json;
stream.avail_in = (uInt)input_size;
stream.next_out = *compressed;
stream.avail_out = (uInt)bound;
status = deflate(&stream, Z_FINISH);
if (status != Z_STREAM_END) {
SetError(error, error_size, "gzip compression failed");
goto done;
}
*compressed_size = (size_t)stream.total_out;
result = 1;
done:
deflateEnd(&stream);
if (!result) {
free(*compressed);
*compressed = NULL;
*compressed_size = 0U;
}
return result;
}
static int
HexDigit(unsigned char value)
{
if (value >= '0' && value <= '9') return value - '0';
value = (unsigned char)tolower(value);
if (value >= 'a' && value <= 'f') return value - 'a' + 10;
return -1;
}
int
BongoWorkerTlsRptDecodeMailto(const char *uri, char *recipient,
size_t recipient_size)
{
const unsigned char *cursor;
size_t length = 0U;
if (!uri || strncmp(uri, "mailto:", 7) != 0 || !recipient ||
recipient_size == 0U) return 0;
cursor = (const unsigned char *)uri + 7U;
while (*cursor && *cursor != '?') {
unsigned char value = *cursor++;
if (value == '%') {
int high = HexDigit(cursor[0]);
int low = HexDigit(cursor[1]);
if (high < 0 || low < 0) return 0;
value = (unsigned char)((high << 4) | low);
cursor += 2;
}
if (value <= 0x20 || value == 0x7f || value == ',' || value == ';' ||
length + 1U >= recipient_size) return 0;
recipient[length++] = (char)value;
}
recipient[length] = '\0';
return length != 0U && strchr(recipient, '@') != NULL;
}
static int
AppendBase64(Buffer *buffer, const unsigned char *data, size_t length)
{
static const char Alphabet[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
size_t index = 0U;
unsigned int column = 0U;
while (index < length) {
size_t remaining = length - index;
unsigned int first = data[index++];
unsigned int second = remaining > 1U ? data[index++] : 0U;
unsigned int third = remaining > 2U ? data[index++] : 0U;
char encoded[4];
encoded[0] = Alphabet[first >> 2];
encoded[1] = Alphabet[((first & 3U) << 4) | (second >> 4)];
encoded[2] = remaining > 1U
? Alphabet[((second & 15U) << 2) | (third >> 6)] : '=';
encoded[3] = remaining > 2U ? Alphabet[third & 63U] : '=';
if (!AppendN(buffer, encoded, sizeof(encoded))) return 0;
column += 4U;
if (column == 76U) {
if (!Append(buffer, "\r\n")) return 0;
column = 0U;
}
}
return column == 0U || Append(buffer, "\r\n");
}
static int
FormatDate(int64_t value, char output[64])
{
static const char *const Days[] =
{"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
static const char *const Months[] =
{"Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
time_t converted = (time_t)value;
struct tm broken_down;
if ((int64_t)converted != value ||
!gmtime_r(&converted, &broken_down) ||
broken_down.tm_wday < 0 || broken_down.tm_wday > 6 ||
broken_down.tm_mon < 0 || broken_down.tm_mon > 11) return 0;
return snprintf(output, 64, "%s, %02d %s %04d %02d:%02d:%02d +0000",
Days[broken_down.tm_wday], broken_down.tm_mday,
Months[broken_down.tm_mon], broken_down.tm_year + 1900,
broken_down.tm_hour, broken_down.tm_min,
broken_down.tm_sec) < 64;
}
int
BongoWorkerTlsRptBuildMessage(const BongoTlsRptDelivery *delivery,
const char *submitter,
const char *from_address,
const char *recipient,
int64_t now,
char **message, size_t *message_size,
char *error, size_t error_size)
{
unsigned char *compressed = NULL;
size_t compressed_size = 0U;
Buffer buffer = {0};
char date[64];
int result = 0;
if (error && error_size) error[0] = '\0';
if (!delivery || !delivery->json || !message || !message_size ||
!HeaderSafe(delivery->report_domain) ||
!HeaderSafe(delivery->report_id) || !HeaderSafe(submitter) ||
!HeaderSafe(from_address) || !HeaderSafe(recipient) ||
delivery->day_start < 0 ||
delivery->day_start > INT64_MAX - 86399 || !FormatDate(now, date)) {
SetError(error, error_size, "unsafe TLSRPT mail metadata");
return 0;
}
*message = NULL;
*message_size = 0U;
if (!BongoWorkerTlsRptGzip(delivery->json, &compressed,
&compressed_size, error, error_size)) return 0;
if (!AppendF(&buffer,
"From: %s\r\nTo: %s\r\nDate: %s\r\n"
"Message-ID: <%s>\r\n"
"Subject: Report Domain: %s Submitter: %s Report-ID: <%s>\r\n"
"TLS-Report-Domain: %s\r\nTLS-Report-Submitter: %s\r\n"
"TLS-Required: No\r\nAuto-Submitted: auto-generated\r\n"
"MIME-Version: 1.0\r\n"
"Content-Type: multipart/report; report-type=\"tlsrpt\"; "
"boundary=\"=_bongo_tlsrpt_%" PRIi64 "\"\r\n\r\n"
"This is a multipart message in MIME format.\r\n\r\n"
"--=_bongo_tlsrpt_%" PRIi64 "\r\n"
"Content-Type: text/plain; charset=us-ascii\r\n"
"Content-Transfer-Encoding: 7bit\r\n\r\n"
"This is an aggregate TLS report from %s.\r\n\r\n"
"--=_bongo_tlsrpt_%" PRIi64 "\r\n"
"Content-Type: application/tlsrpt+gzip\r\n"
"Content-Transfer-Encoding: base64\r\n"
"Content-Disposition: attachment; filename=\"%s!%s!%" PRIi64
"!%" PRIi64 "!%" PRIi64 ".json.gz\"\r\n\r\n",
from_address, recipient, date, delivery->report_id,
delivery->report_domain, submitter, delivery->report_id,
delivery->report_domain, submitter, delivery->id, delivery->id,
submitter, delivery->id, submitter, delivery->report_domain,
delivery->day_start, delivery->day_start + 86399, delivery->id) ||
!AppendBase64(&buffer, compressed, compressed_size) ||
!AppendF(&buffer, "\r\n--=_bongo_tlsrpt_%" PRIi64 "--\r\n",
delivery->id)) {
SetError(error, error_size, "out of memory building TLSRPT message");
goto done;
}
*message = buffer.data;
*message_size = buffer.length;
buffer.data = NULL;
result = 1;
done:
free(compressed);
free(buffer.data);
return result;
}
int64_t
BongoWorkerTlsRptRetryDelay(int attempts)
{
int64_t delay = 300;
if (attempts < 0) attempts = 0;
while (attempts-- > 0 && delay < 21600) delay *= 2;
return delay > 21600 ? 21600 : delay;
}
BongoTlsRptDeliveryResult
BongoWorkerTlsRptHttpResult(long status)
{
if (status >= 200 && status < 300) return BONGO_TLSRPT_DELIVERY_SUCCESS;
if (status == 408 || status == 425 || status == 429 || status >= 500)
return BONGO_TLSRPT_DELIVERY_RETRY;
return BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE;
}
@@ -0,0 +1,33 @@
/****************************************************************************
* <Novell-copyright>
* Copyright (c) 2001 Novell, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 2 of the GNU General Public License.
* </Novell-copyright>
****************************************************************************/
#ifndef BONGO_WORKER_TLSRPT_DELIVERY_SUPPORT_H
#define BONGO_WORKER_TLSRPT_DELIVERY_SUPPORT_H
#include <bongotlsrpt.h>
#include <stddef.h>
#include <stdint.h>
int BongoWorkerTlsRptGzip(const char *json, unsigned char **compressed,
size_t *compressed_size,
char *error, size_t error_size);
int BongoWorkerTlsRptDecodeMailto(const char *uri, char *recipient,
size_t recipient_size);
int BongoWorkerTlsRptBuildMessage(const BongoTlsRptDelivery *delivery,
const char *submitter,
const char *from_address,
const char *recipient,
int64_t now,
char **message, size_t *message_size,
char *error, size_t error_size);
int64_t BongoWorkerTlsRptRetryDelay(int attempts);
BongoTlsRptDeliveryResult BongoWorkerTlsRptHttpResult(long status);
#endif
+448
View File
@@ -0,0 +1,448 @@
/****************************************************************************
* <Novell-copyright>
* Copyright (c) 2001 Novell, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 2 of the GNU General Public License.
* </Novell-copyright>
****************************************************************************/
#include <config.h>
#include "worker.h"
#include "tlsrpt-delivery-support.h"
#include <bongojson.h>
#include <bongotlsrpt.h>
#include <bongoutil.h>
#include <curl/curl.h>
#include <logger.h>
#include <memmgr.h>
#include <nmap.h>
#include <nmlib.h>
#include <xpl.h>
#include <inttypes.h>
#include <limits.h>
#include <stdint.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#define TLSRPT_MAX_BATCHES 64
#define TLSRPT_MAX_DELIVERIES 64
#define TLSRPT_INITIAL_DELAY_MAX 14400
typedef struct TlsRptConfiguration {
BongoJsonNode *smtp_node;
BongoJsonNode *global_node;
BongoJsonNode *worker_node;
int enabled;
int dkim_enabled;
const char *database;
const char *hostname;
const char *dkim_domain;
const char *dkim_selector;
const char *dkim_key_directory;
const char *organization_name;
char submitter[BONGO_TLSRPT_DOMAIN_SIZE];
char from_address[BONGO_TLSRPT_DOMAIN_SIZE * 2];
char contact_info[BONGO_TLSRPT_URI_SIZE];
} TlsRptConfiguration;
static int
ReadDocument(const char *name, BongoJsonNode **node,
char *error, size_t error_size)
{
char *content = NULL;
int result = 0;
*node = NULL;
if (!NMAPReadConfigFile(name, &content) || !content ||
BongoJsonParseString(content, node) != BONGO_JSON_OK || !*node ||
(*node)->type != BONGO_JSON_OBJECT) {
snprintf(error, error_size,
"cannot read valid %s configuration from Store", name);
goto done;
}
result = 1;
done:
if (content) MemFree(content);
if (!result && *node) {
BongoJsonNodeFree(*node);
*node = NULL;
}
return result;
}
static void
FreeConfiguration(TlsRptConfiguration *configuration)
{
if (configuration->global_node)
BongoJsonNodeFree(configuration->global_node);
if (configuration->worker_node)
BongoJsonNodeFree(configuration->worker_node);
if (configuration->smtp_node)
BongoJsonNodeFree(configuration->smtp_node);
memset(configuration, 0, sizeof(*configuration));
}
static int
SafePath(const char *path)
{
size_t length;
if (!path || path[0] != '/') return 0;
length = strlen(path);
return length > 1U && length < PATH_MAX && !strstr(path, "/../") &&
!(length >= 3U && !strcmp(path + length - 3U, "/.."));
}
static int
LoadConfiguration(TlsRptConfiguration *configuration,
char *error, size_t error_size)
{
BongoJsonObject *smtp;
BongoJsonObject *global;
BongoJsonObject *worker;
BongoJsonObject *tls_reporting;
BOOL enabled = FALSE;
BOOL dkim_enabled = FALSE;
const char *domain;
memset(configuration, 0, sizeof(*configuration));
if (!ReadDocument("smtp", &configuration->smtp_node,
error, error_size) ||
!ReadDocument("global", &configuration->global_node,
error, error_size) ||
!ReadDocument("worker", &configuration->worker_node,
error, error_size)) return 0;
smtp = BongoJsonNodeAsObject(configuration->smtp_node);
global = BongoJsonNodeAsObject(configuration->global_node);
worker = BongoJsonNodeAsObject(configuration->worker_node);
if (BongoJsonObjectGetBool(smtp, "outbound_tlsrpt_enabled", &enabled) !=
BONGO_JSON_OK ||
BongoJsonObjectGetString(smtp, "outbound_tlsrpt_database",
&configuration->database) != BONGO_JSON_OK ||
BongoJsonObjectGetBool(smtp, "dkim_sign_outgoing", &dkim_enabled) !=
BONGO_JSON_OK ||
BongoJsonObjectGetString(smtp, "dkim_signing_domain",
&configuration->dkim_domain) != BONGO_JSON_OK ||
BongoJsonObjectGetString(smtp, "dkim_selector",
&configuration->dkim_selector) != BONGO_JSON_OK ||
BongoJsonObjectGetString(smtp, "dkim_key_directory",
&configuration->dkim_key_directory) !=
BONGO_JSON_OK ||
BongoJsonObjectGetString(global, "hostname",
&configuration->hostname) != BONGO_JSON_OK ||
BongoJsonObjectGetObject(worker, "tls_reporting", &tls_reporting) !=
BONGO_JSON_OK ||
BongoJsonObjectGetString(tls_reporting, "organization_name",
&configuration->organization_name) !=
BONGO_JSON_OK || !configuration->organization_name ||
!configuration->organization_name[0] ||
strlen(configuration->organization_name) > 1023U ||
!SafePath(configuration->database)) {
snprintf(error, error_size, "invalid TLSRPT worker configuration");
return 0;
}
configuration->enabled = enabled != FALSE;
configuration->dkim_enabled = dkim_enabled != FALSE;
domain = configuration->dkim_domain && configuration->dkim_domain[0]
? configuration->dkim_domain : configuration->hostname;
if (!BongoDomainToASCII(domain, configuration->submitter,
sizeof(configuration->submitter)) ||
snprintf(configuration->from_address,
sizeof(configuration->from_address), "tlsrpt@%s",
configuration->submitter) >=
(int)sizeof(configuration->from_address) ||
snprintf(configuration->contact_info,
sizeof(configuration->contact_info), "mailto:%s",
configuration->from_address) >=
(int)sizeof(configuration->contact_info)) {
snprintf(error, error_size, "invalid TLSRPT reporting domain");
return 0;
}
return 1;
}
static size_t
DiscardResponse(char *data, size_t size, size_t count, void *context)
{
(void)data;
(void)context;
if (size != 0U && count > SIZE_MAX / size) return 0U;
return size * count;
}
static BongoTlsRptDeliveryResult
DeliverHttps(const BongoTlsRptDelivery *delivery,
char *error, size_t error_size)
{
unsigned char *compressed = NULL;
size_t compressed_size = 0U;
struct curl_slist *headers = NULL;
CURL *curl = NULL;
CURLcode status;
long response = 0;
BongoTlsRptDeliveryResult result = BONGO_TLSRPT_DELIVERY_RETRY;
if (!BongoWorkerTlsRptGzip(delivery->json, &compressed, &compressed_size,
error, error_size)) return result;
curl = curl_easy_init();
headers = curl_slist_append(headers,
"Content-Type: application/tlsrpt+gzip");
if (!curl || !headers) {
snprintf(error, error_size, "cannot initialize HTTPS delivery");
goto done;
}
curl_easy_setopt(curl, CURLOPT_URL, delivery->endpoint_uri);
#if LIBCURL_VERSION_NUM >= 0x075500
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
#else
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, (long)CURLPROTO_HTTPS);
#endif
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, compressed);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
(curl_off_t)compressed_size);
curl_easy_setopt(curl, CURLOPT_USERAGENT,
"Bongo/" BONGO_VERSION " TLSRPT");
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, DiscardResponse);
status = curl_easy_perform(curl);
if (status != CURLE_OK) {
snprintf(error, error_size, "HTTPS delivery failed: %s",
curl_easy_strerror(status));
if (status == CURLE_UNSUPPORTED_PROTOCOL || status == CURLE_URL_MALFORMAT)
result = BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE;
goto done;
}
if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response) != CURLE_OK) {
snprintf(error, error_size, "cannot read HTTPS response status");
goto done;
}
result = BongoWorkerTlsRptHttpResult(response);
if (result != BONGO_TLSRPT_DELIVERY_SUCCESS)
snprintf(error, error_size, "HTTPS endpoint returned status %ld",
response);
done:
curl_slist_free_all(headers);
if (curl) curl_easy_cleanup(curl);
free(compressed);
return result;
}
static int
QueueCommand(Connection *connection, char *response, size_t response_size,
const char *format, ...)
{
va_list arguments;
int written;
va_start(arguments, format);
written = ConnWriteVF(connection, format, arguments);
va_end(arguments);
if (written < 0 || ConnFlush(connection) < 0) return -1;
return NMAPReadResponse(connection, response, response_size, TRUE);
}
static BongoTlsRptDeliveryResult
DeliverMail(const TlsRptConfiguration *configuration,
const BongoTlsRptDelivery *delivery,
int64_t now, char *error, size_t error_size)
{
Connection *connection = NULL;
char recipient[BONGO_TLSRPT_URI_SIZE];
char response[CONN_BUFSIZE];
char key_path[PATH_MAX];
char *message = NULL;
size_t message_size = 0U;
BongoTlsRptDeliveryResult result = BONGO_TLSRPT_DELIVERY_RETRY;
if (!BongoWorkerTlsRptDecodeMailto(delivery->endpoint_uri, recipient,
sizeof(recipient))) {
snprintf(error, error_size, "invalid TLSRPT mailto endpoint");
return BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE;
}
if (!configuration->dkim_enabled || !configuration->dkim_selector ||
!configuration->dkim_selector[0] ||
!configuration->dkim_key_directory ||
snprintf(key_path, sizeof(key_path), "%s/%s/%s.private",
configuration->dkim_key_directory, configuration->submitter,
configuration->dkim_selector) >= (int)sizeof(key_path) ||
access(key_path, R_OK) != 0) {
snprintf(error, error_size,
"RFC 8460 mail delivery requires a readable DKIM key for %s",
configuration->submitter);
return result;
}
if (!BongoWorkerTlsRptBuildMessage(
delivery, configuration->submitter, configuration->from_address,
recipient, now, &message, &message_size, error, error_size))
return result;
connection = NMAPConnectQueue("127.0.0.1", NULL);
if (!connection ||
!NMAPAuthenticateToQueue(connection, response, sizeof(response))) {
snprintf(error, error_size, "cannot authenticate to Bongo queue");
goto done;
}
if (QueueCommand(connection, response, sizeof(response), "QCREA\r\n") !=
1000 ||
NMAPSendCommandF(connection, "QSTOR MESSAGE %" PRIu64 "\r\n",
(uint64_t)message_size) < 0 ||
NMAPSendCommand(connection, message, message_size) < 0 ||
NMAPReadResponse(connection, response, sizeof(response), TRUE) != 1000 ||
QueueCommand(connection, response, sizeof(response),
"QSTOR TO %s %s 2\r\n", recipient, recipient) != 1000 ||
QueueCommand(connection, response, sizeof(response),
"QSTOR FROM %s -\r\n",
configuration->from_address) != 1000 ||
QueueCommand(connection, response, sizeof(response),
"QSTOR FLAGS %d\r\n", MSG_FLAG_TLS_REPORT) != 1000 ||
QueueCommand(connection, response, sizeof(response), "QRUN\r\n") !=
1000) {
snprintf(error, error_size, "Bongo queue rejected TLSRPT report");
goto done;
}
result = BONGO_TLSRPT_DELIVERY_SUCCESS;
done:
if (connection) NMAPQuit(connection);
free(message);
return result;
}
static unsigned int
InitialDelay(int64_t now)
{
unsigned int seed = (unsigned int)now ^
(unsigned int)((uint64_t)now >> 32) ^ (unsigned int)getpid();
return 1U + rand_r(&seed) % TLSRPT_INITIAL_DELAY_MAX;
}
int
BongoWorkerRunTlsRptDelivery(char *error, size_t error_size,
int64_t *not_before)
{
TlsRptConfiguration configuration;
BongoTlsRptStore *store = NULL;
BongoTlsRptDelivery delivery;
int64_t now = (int64_t)time(NULL);
int64_t today;
int64_t next_due = 0;
int prepared;
int delivered;
int curl_initialized = 0;
int result = 0;
if (error && error_size) error[0] = '\0';
if (not_before) *not_before = 0;
memset(&configuration, 0, sizeof(configuration));
if (now < 0 || !LoadConfiguration(&configuration, error, error_size)) {
FreeConfiguration(&configuration);
return 0;
}
if (!configuration.enabled) {
FreeConfiguration(&configuration);
return 1;
}
if (!BongoTlsRptStoreOpen(&store, configuration.database)) {
snprintf(error, error_size, "cannot open TLSRPT database %s",
configuration.database);
goto done;
}
if (!BongoTlsRptStoreCleanupExpired(store, now)) {
snprintf(error, error_size, "cannot expire old TLSRPT deliveries");
goto done;
}
today = (now / 86400) * 86400;
for (prepared = 0; prepared < TLSRPT_MAX_BATCHES; prepared++) {
int status = BongoTlsRptStorePrepareNext(
store, today, configuration.organization_name,
configuration.contact_info,
configuration.submitter, now, now + InitialDelay(now + prepared));
if (status < 0) {
snprintf(error, error_size, "cannot prepare TLSRPT report batch");
goto done;
}
if (status == 0) break;
}
if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) {
snprintf(error, error_size, "cannot initialize HTTPS client");
goto done;
}
curl_initialized = 1;
for (delivered = 0; delivered < TLSRPT_MAX_DELIVERIES; delivered++) {
BongoTlsRptDeliveryResult delivery_result;
int status = BongoTlsRptStoreNextDelivery(store, now, &delivery,
&next_due);
char delivery_error[BONGO_WORKER_ERROR_SIZE] = {0};
int64_t retry_at = 0;
if (status < 0) {
snprintf(error, error_size, "cannot read pending TLSRPT delivery");
goto done;
}
if (status == 0) break;
if (delivery.endpoint_type == BONGO_TLSRPT_ENDPOINT_MAILTO)
delivery_result = DeliverMail(&configuration, &delivery, now,
delivery_error,
sizeof(delivery_error));
else if (delivery.endpoint_type == BONGO_TLSRPT_ENDPOINT_HTTPS)
delivery_result = DeliverHttps(&delivery, delivery_error,
sizeof(delivery_error));
else {
delivery_result = BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE;
snprintf(delivery_error, sizeof(delivery_error),
"unsupported TLSRPT endpoint type");
}
if (delivery_result == BONGO_TLSRPT_DELIVERY_RETRY) {
retry_at = now + BongoWorkerTlsRptRetryDelay(delivery.attempts);
if (retry_at >= delivery.expires_at)
delivery_result = BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE;
}
if (!BongoTlsRptStoreFinishDelivery(
store, delivery.id, delivery_result, retry_at,
delivery_error)) {
BongoTlsRptDeliveryFree(&delivery);
snprintf(error, error_size, "cannot persist TLSRPT delivery state");
goto done;
}
if (delivery_result != BONGO_TLSRPT_DELIVERY_SUCCESS)
Log(LOG_WARNING, "TLSRPT delivery to %.400s failed: %.400s",
delivery.endpoint_uri,
delivery_error[0] ? delivery_error : "unknown error");
BongoTlsRptDeliveryFree(&delivery);
}
curl_global_cleanup();
curl_initialized = 0;
{
int pending = BongoTlsRptStoreNextDelivery(store, now, &delivery,
&next_due);
if (pending < 0) {
snprintf(error, error_size,
"cannot schedule pending TLSRPT delivery");
goto done;
}
if (pending == 1) next_due = now + 1;
}
BongoTlsRptDeliveryFree(&delivery);
if (next_due != 0 && not_before)
*not_before = next_due > now ? next_due : now + 1;
result = 1;
done:
if (curl_initialized) curl_global_cleanup();
BongoTlsRptStoreClose(&store);
FreeConfiguration(&configuration);
return result;
}
@@ -0,0 +1,80 @@
/****************************************************************************
* <Novell-copyright>
* Copyright (c) 2001 Novell, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 2 of the GNU General Public License.
* </Novell-copyright>
****************************************************************************/
#include "tlsrpt-delivery-support.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>
int
main(void)
{
BongoTlsRptDelivery delivery = {
.id = 42,
.endpoint_type = BONGO_TLSRPT_ENDPOINT_MAILTO,
.day_start = 172800,
.created_at = 300000,
.expires_at = 386400,
.report_domain = "receiver.example",
.report_id = "tlsrpt-172800-42@mail.sender.example",
.endpoint_uri = "mailto:tls%2brpt@receiver.example",
.json = "{\"organization-name\":\"Bongo\"}"
};
unsigned char *compressed = NULL;
size_t compressed_size = 0U;
char recipient[256];
char error[256];
char *message = NULL;
size_t message_size = 0U;
assert(BongoWorkerTlsRptGzip(delivery.json, &compressed,
&compressed_size, error, sizeof(error)));
assert(compressed_size > 18U);
assert(compressed[0] == 0x1f && compressed[1] == 0x8b);
free(compressed);
assert(BongoWorkerTlsRptDecodeMailto(delivery.endpoint_uri, recipient,
sizeof(recipient)));
assert(strcmp(recipient, "tls+rpt@receiver.example") == 0);
assert(!BongoWorkerTlsRptDecodeMailto(
"mailto:reports%0d%0aBcc:x@example.test", recipient,
sizeof(recipient)));
assert(BongoWorkerTlsRptBuildMessage(
&delivery, "mail.sender.example", "tlsrpt@mail.sender.example",
"tls+rpt@receiver.example", 300000, &message, &message_size,
error, sizeof(error)));
assert(message_size == strlen(message));
assert(strstr(message, "TLS-Required: No\r\n") != NULL);
assert(strstr(message,
"Content-Type: application/tlsrpt+gzip\r\n") != NULL);
assert(strstr(message,
"boundary=\"=_bongo_tlsrpt_42\"\r\n") != NULL);
assert(strstr(message,
"TLS-Report-Domain: receiver.example\r\n") != NULL);
assert(strstr(message,
"TLS-Report-Submitter: mail.sender.example\r\n") != NULL);
assert(strstr(message,
"mail.sender.example!receiver.example!172800!259199!42.json.gz") !=
NULL);
free(message);
assert(BongoWorkerTlsRptRetryDelay(0) == 300);
assert(BongoWorkerTlsRptRetryDelay(1) == 600);
assert(BongoWorkerTlsRptRetryDelay(20) == 21600);
assert(BongoWorkerTlsRptHttpResult(200) ==
BONGO_TLSRPT_DELIVERY_SUCCESS);
assert(BongoWorkerTlsRptHttpResult(429) ==
BONGO_TLSRPT_DELIVERY_RETRY);
assert(BongoWorkerTlsRptHttpResult(503) ==
BONGO_TLSRPT_DELIVERY_RETRY);
assert(BongoWorkerTlsRptHttpResult(404) ==
BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE);
return 0;
}
+7 -2
View File
@@ -42,6 +42,9 @@ main(void)
"\"retry_initial_seconds\":300,\"retry_max_seconds\":21600},{"
"\"name\":\"scanner-health\",\"enabled\":true,"
"\"interval_seconds\":21600,\"jitter_seconds\":900,"
"\"retry_initial_seconds\":300,\"retry_max_seconds\":21600},{"
"\"name\":\"tlsrpt-delivery\",\"enabled\":true,"
"\"interval_seconds\":3600,\"jitter_seconds\":300,"
"\"retry_initial_seconds\":300,\"retry_max_seconds\":21600}]}";
static const char Unknown[] =
"{\"poll_interval_seconds\":30,\"jobs\":[{"
@@ -57,12 +60,14 @@ main(void)
failed |= Check(BongoWorkerParseConfiguration(
Valid, &poll, &jobs, &count, error, sizeof(error)),
"valid worker configuration was rejected");
failed |= Check(!failed && poll == 30 && count == 2 &&
failed |= Check(!failed && poll == 30 && count == 3 &&
strcmp(jobs[0].name, "acme-renew") == 0 &&
jobs[0].enabled == 0 &&
jobs[0].interval_seconds == 43200 &&
strcmp(jobs[1].name, "scanner-health") == 0 &&
jobs[1].enabled != 0,
jobs[1].enabled != 0 &&
strcmp(jobs[2].name, "tlsrpt-delivery") == 0 &&
jobs[2].enabled != 0,
"valid worker configuration was parsed incorrectly");
BongoWorkerFreeConfiguration(&jobs, count);
count = 0;
+11
View File
@@ -11,6 +11,9 @@
"/var/db/spamassassin"
]
},
"tls_reporting": {
"organization_name": "Bongo"
},
"jobs": [
{
"name": "acme-renew",
@@ -27,6 +30,14 @@
"jitter_seconds": 900,
"retry_initial_seconds": 300,
"retry_max_seconds": 21600
},
{
"name": "tlsrpt-delivery",
"enabled": true,
"interval_seconds": 3600,
"jitter_seconds": 300,
"retry_initial_seconds": 300,
"retry_max_seconds": 21600
}
]
}
@@ -177,6 +177,25 @@ class ConfigurationModelTest(unittest.TestCase):
self.assertTrue(any("outbound_mta_sts_cache_directory" in message
for message in messages))
def test_tls_reporting_requires_registered_job_and_safe_organization(self):
defaults = templates()
configs = apply_scenario(defaults, Scenario(
hostname="mail.example.test", bind_address="192.0.2.10",
domains=["example.test"], admin_password="0123456789ab",
web_mode="direct", web_public_url="http://mail.example.test:8080",
))
configs["worker"]["tls_reporting"]["organization_name"] = "bad\nname"
configs["worker"]["jobs"] = [
job for job in configs["worker"]["jobs"]
if job["name"] != "tlsrpt-delivery"
]
messages = [str(issue) for issue in
validate_all(configs, defaults, False)]
self.assertTrue(any("organization_name" in message
for message in messages))
self.assertTrue(any("tlsrpt-delivery" in message
for message in messages))
def test_direct_https_binds_443_and_redirects_80(self):
defaults = templates()
configs = apply_scenario(defaults, Scenario(
+1 -1
View File
@@ -171,7 +171,7 @@ class ConfigurationStorageTest(unittest.TestCase):
result = merge_defaults(configurations, self.store.templates)
self.assertEqual(
[job["name"] for job in result["worker"]["jobs"]],
["acme-renew", "scanner-health"],
["acme-renew", "tlsrpt-delivery", "scanner-health"],
)
def test_migration_moves_historical_files_and_writes_all_documents(self):
+31 -1
View File
@@ -65,7 +65,8 @@ _HEADER_NAME = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
_ATTRIBUTE_NAME = re.compile(r"^[A-Za-z0-9.-]{1,63}$")
_SELECTOR = re.compile(r"^[A-Za-z0-9_-]{1,63}$")
_DEVICE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
_WORKER_JOB_NAMES = frozenset({"acme-renew", "scanner-health"})
_WORKER_JOB_NAMES = frozenset({"acme-renew", "scanner-health",
"tlsrpt-delivery"})
def _safe_absolute_path(value: Any) -> bool:
@@ -691,6 +692,18 @@ def _validate_worker(config: Mapping[str, Any], issues: list[Issue]) -> None:
_issue(issues, "worker", path + ".retry_max_seconds",
"maximum retry delay must not be shorter than the initial delay")
scanner_health = config.get("scanner_health")
tls_reporting = config.get("tls_reporting")
if not isinstance(tls_reporting, dict):
_issue(issues, "worker", "tls_reporting",
"expected a TLS reporting object")
else:
organization = tls_reporting.get("organization_name")
if (not isinstance(organization, str) or not organization or
len(organization.encode("utf-8")) > 1023 or
any(ord(character) < 0x20 or ord(character) == 0x7f
for character in organization)):
_issue(issues, "worker", "tls_reporting.organization_name",
"expected a non-empty control-free name of at most 1023 bytes")
if not isinstance(scanner_health, dict):
_issue(issues, "worker", "scanner_health",
"expected a scanner health object")
@@ -796,6 +809,22 @@ def _validate_acme(configurations: Mapping[str, Any],
"disable the acme-renew job when ACME is disabled")
def _validate_tlsrpt(configurations: Mapping[str, Any], issues: list[Issue]) -> None:
smtp = configurations.get("smtp", {})
if smtp.get("outbound_tlsrpt_enabled") is not True:
return
worker = configurations.get("worker", {})
jobs = [job for job in worker.get("jobs", [])
if isinstance(job, dict) and job.get("name") == "tlsrpt-delivery"]
if len(jobs) != 1 or jobs[0].get("enabled") is not True:
_issue(issues, "worker", "jobs",
"outbound TLS reporting requires the tlsrpt-delivery worker job")
if smtp.get("dkim_sign_outgoing") is not True:
_issue(issues, "smtp", "dkim_sign_outgoing",
"TLS reports sent to mailto endpoints require DKIM signing",
"warning")
def _validate_scanners(configurations: Mapping[str, Any], issues: list[Issue]) -> None:
manager = configurations.get("manager", {})
enabled_agents = {
@@ -994,6 +1023,7 @@ def validate_all(configurations: Mapping[str, Any],
_validate_manager(configurations.get("manager", {}), issues)
_validate_acme(configurations, issues)
_validate_worker(configurations.get("worker", {}), issues)
_validate_tlsrpt(configurations, issues)
_validate_scanners(configurations, issues)
_validate_web(configurations.get("web", {}), issues, check_files)
_validate_auth(configurations.get("authentication", {}), issues, check_files)
+2 -2
View File
@@ -60,9 +60,9 @@ main(void)
failed |= Check(!BongoTlsRptParseRecord(
"v=TLSRPTv1; x-future=contains space; rua=mailto:r@example.test",
&policy), "invalid extension value was accepted");
failed |= Check(!BongoTlsRptParseRecord(
failed |= Check(BongoTlsRptParseRecord(
"v=TLSRPTv1; rua=mailto:r!size@example.test", &policy),
"unencoded mailto URI delimiter was accepted");
"RFC 6068 mailto local-part delimiter was rejected");
failed |= Check(!BongoTlsRptParseRecord(
" v=TLSRPTv1; rua=mailto:r@example.test", &policy),
"leading whitespace before the version was accepted");
+63
View File
@@ -48,7 +48,10 @@ main(void)
"https://errors.example.test/cert", "certificate chain did not validate"
};
BongoJsonNode *parsed = NULL;
BongoTlsRptDelivery delivery;
char first_report_id[BONGO_TLSRPT_DOMAIN_SIZE * 2] = "";
char *json = NULL;
int64_t next_not_before = 0;
int descriptor;
int failed = 0;
@@ -85,6 +88,66 @@ main(void)
"v=TLSRPTv1\t; rua=mailto:reports@example.test",
"Bongo test", "mailto:postmaster@example.test", "empty", &json),
"delivered TLSRPT rows were not removed");
failed |= Check(!failed && BongoTlsRptStoreRecord(store, &success) &&
BongoTlsRptStorePrepareNext(store, 259200, "Bongo test",
"mailto:postmaster@example.test", "mail.example.test",
300000, 300100) == 1,
"cannot atomically prepare a durable TLSRPT delivery");
failed |= Check(!failed &&
BongoTlsRptStoreNextDelivery(store, 300000, &delivery,
&next_not_before) == 0 &&
next_not_before == 300100,
"TLSRPT delivery ignored its initial random delay");
/* A late event for the same day and policy must become a new report,
* never be deleted with the already prepared snapshot. */
failed |= Check(!failed && BongoTlsRptStoreRecord(store, &success) &&
BongoTlsRptStorePrepareNext(store, 259200, "Bongo test",
"mailto:postmaster@example.test", "mail.example.test",
300001, 300100) == 1 &&
BongoTlsRptStorePrepareNext(store, 259200, "Bongo test",
"mailto:postmaster@example.test", "mail.example.test",
300001, 300100) == 0,
"late TLSRPT event was lost or prepared more than once");
failed |= Check(!failed &&
BongoTlsRptStoreNextDelivery(store, 300100, &delivery,
&next_not_before) == 1 &&
delivery.endpoint_type == BONGO_TLSRPT_ENDPOINT_MAILTO &&
delivery.attempts == 0 && delivery.expires_at == 386500 &&
delivery.json &&
strstr(delivery.json, "\"total-successful-session-count\":1"),
"prepared TLSRPT delivery is invalid");
if (delivery.report_id[0])
snprintf(first_report_id, sizeof(first_report_id), "%s",
delivery.report_id);
failed |= Check(!failed && BongoTlsRptStoreFinishDelivery(
store, delivery.id, BONGO_TLSRPT_DELIVERY_RETRY, 300200,
"temporary queue failure"), "cannot defer TLSRPT delivery");
BongoTlsRptDeliveryFree(&delivery);
failed |= Check(!failed &&
BongoTlsRptStoreNextDelivery(store, 300100, &delivery,
&next_not_before) == 1 &&
strcmp(delivery.report_id, first_report_id) != 0 &&
BongoTlsRptStoreFinishDelivery(store, delivery.id,
BONGO_TLSRPT_DELIVERY_SUCCESS, 0, ""),
"late TLSRPT event did not produce an independent report");
BongoTlsRptDeliveryFree(&delivery);
failed |= Check(!failed &&
BongoTlsRptStoreNextDelivery(store, 300200, &delivery,
&next_not_before) == 1 &&
delivery.attempts == 1 &&
strcmp(delivery.report_id, first_report_id) == 0 &&
BongoTlsRptStoreFinishDelivery(store, delivery.id,
BONGO_TLSRPT_DELIVERY_SUCCESS, 0, ""),
"TLSRPT retry state was not persisted");
BongoTlsRptDeliveryFree(&delivery);
failed |= Check(!failed &&
BongoTlsRptStoreNextDelivery(store, 300200, &delivery,
&next_not_before) == 0 &&
next_not_before == 0 &&
BongoTlsRptStoreCleanupExpired(store, 400000),
"completed TLSRPT reports were not retired");
BongoTlsRptStoreClose(&store);
unlink(database);
snprintf(sidecar, sizeof(sidecar), "%s-wal", database);
+317 -3
View File
@@ -23,6 +23,7 @@
#include <sqlite3.h>
#include <ctype.h>
#include <inttypes.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
@@ -34,6 +35,7 @@ struct BongoTlsRptStore {
};
static const char StoreSchema[] =
"PRAGMA foreign_keys=ON;"
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=FULL;"
"CREATE TABLE IF NOT EXISTS tlsrpt_events ("
@@ -59,7 +61,30 @@ static const char StoreSchema[] =
" receiving_ip,additional_information,failure_reason)"
") WITHOUT ROWID;"
"CREATE INDEX IF NOT EXISTS tlsrpt_events_report "
"ON tlsrpt_events(report_domain,day_start);";
"ON tlsrpt_events(report_domain,day_start);"
"CREATE TABLE IF NOT EXISTS tlsrpt_reports ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" report_domain TEXT NOT NULL,"
" day_start INTEGER NOT NULL,"
" reporting_policy TEXT NOT NULL,"
" report_id TEXT NOT NULL UNIQUE,"
" json TEXT NOT NULL,"
" created_at INTEGER NOT NULL,"
" expires_at INTEGER NOT NULL"
");"
"CREATE TABLE IF NOT EXISTS tlsrpt_deliveries ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" report INTEGER NOT NULL REFERENCES tlsrpt_reports(id) ON DELETE CASCADE,"
" endpoint_type INTEGER NOT NULL,"
" endpoint_uri TEXT NOT NULL,"
" state INTEGER NOT NULL DEFAULT 0,"
" attempts INTEGER NOT NULL DEFAULT 0,"
" next_attempt INTEGER NOT NULL,"
" last_error TEXT NOT NULL DEFAULT '',"
" UNIQUE(report,endpoint_uri)"
");"
"CREATE INDEX IF NOT EXISTS tlsrpt_deliveries_due "
"ON tlsrpt_deliveries(state,next_attempt);";
static char *
Trim(char *value)
@@ -98,8 +123,8 @@ ParseEndpoint(char *value, BongoTlsRptEndpoint *endpoint)
if (!ValidUriText(value)) return 0;
if (strncmp(value, "mailto:", 7) == 0) {
/* RFC 8460 requires URI delimiters in an address to be encoded. */
if (!value[7] || !strchr(value + 7, '@') || strchr(value + 7, '!') ||
strchr(value + 7, ',') || strchr(value + 7, ';')) return 0;
if (!value[7] || !strchr(value + 7, '@') || strchr(value + 7, ',') ||
strchr(value + 7, ';')) return 0;
endpoint->type = BONGO_TLSRPT_ENDPOINT_MAILTO;
} else if (strncmp(value, "https://", 8) == 0) {
authority = value + 8;
@@ -635,3 +660,292 @@ BongoTlsRptStoreDeleteReport(BongoTlsRptStore *store,
sqlite3_finalize(statement);
return result;
}
static int
BeginImmediate(BongoTlsRptStore *store)
{
return sqlite3_exec(store->database, "BEGIN IMMEDIATE", NULL, NULL,
NULL) == SQLITE_OK;
}
static int
Commit(BongoTlsRptStore *store)
{
return sqlite3_exec(store->database, "COMMIT", NULL, NULL, NULL) ==
SQLITE_OK;
}
static void
Rollback(BongoTlsRptStore *store)
{
(void)sqlite3_exec(store->database, "ROLLBACK", NULL, NULL, NULL);
}
static int
CopyColumn(sqlite3_stmt *statement, int column, char *output, size_t size)
{
const unsigned char *value = sqlite3_column_text(statement, column);
size_t length;
if (!value || !output || size == 0U) return 0;
length = strlen((const char *)value);
if (length >= size) return 0;
memcpy(output, value, length + 1U);
return 1;
}
int
BongoTlsRptStorePrepareNext(BongoTlsRptStore *store,
int64_t before_day_start,
const char *organization_name,
const char *contact_info,
const char *submitter,
int64_t now,
int64_t initial_not_before)
{
static const char Select[] =
"SELECT report_domain,day_start,reporting_policy "
"FROM tlsrpt_events WHERE day_start<? "
"GROUP BY report_domain,day_start,reporting_policy "
"ORDER BY day_start,report_domain,reporting_policy LIMIT 1";
static const char InsertReport[] =
"INSERT INTO tlsrpt_reports(report_domain,day_start,"
"reporting_policy,report_id,json,created_at,expires_at) "
"VALUES(?,?,?,'','',?,?)";
static const char UpdateReport[] =
"UPDATE tlsrpt_reports SET report_id=?,json=? WHERE id=?";
static const char InsertDelivery[] =
"INSERT OR IGNORE INTO tlsrpt_deliveries(report,endpoint_type,"
"endpoint_uri,next_attempt) VALUES(?,?,?,?)";
sqlite3_stmt *statement = NULL;
BongoTlsRptDnsPolicy policy;
char report_domain[BONGO_TLSRPT_DOMAIN_SIZE];
char reporting_policy[BONGO_TLSRPT_REPORTING_POLICY_SIZE];
char report_id[BONGO_TLSRPT_DOMAIN_SIZE * 2];
char *json = NULL;
int64_t day_start;
int64_t report;
size_t index;
int result = -1;
if (!store || before_day_start < 0 || before_day_start % 86400 != 0 ||
now < 0 || initial_not_before < now ||
initial_not_before > INT64_MAX - 86400 ||
!ValidText(organization_name, 1024, 0) || !*organization_name ||
!ValidText(contact_info, BONGO_TLSRPT_URI_SIZE, 0) || !*contact_info ||
!ValidText(submitter, BONGO_TLSRPT_DOMAIN_SIZE, 0) || !*submitter ||
!BeginImmediate(store)) return -1;
if (sqlite3_prepare_v2(store->database, Select, -1, &statement, NULL) !=
SQLITE_OK) goto done;
sqlite3_bind_int64(statement, 1, before_day_start);
{
int step = sqlite3_step(statement);
if (step != SQLITE_ROW) {
if (step == SQLITE_DONE) result = 0;
goto done;
}
}
if (!CopyColumn(statement, 0, report_domain, sizeof(report_domain)) ||
!CopyColumn(statement, 2, reporting_policy,
sizeof(reporting_policy))) goto done;
day_start = sqlite3_column_int64(statement, 1);
sqlite3_finalize(statement);
statement = NULL;
if (!BongoTlsRptParseRecord(reporting_policy, &policy)) goto done;
if (sqlite3_prepare_v2(store->database, InsertReport, -1, &statement,
NULL) != SQLITE_OK) goto done;
sqlite3_bind_text(statement, 1, report_domain, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(statement, 2, day_start);
sqlite3_bind_text(statement, 3, reporting_policy, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(statement, 4, now);
/* RFC 8460 permits retries for 24 hours after the initial attempt. */
sqlite3_bind_int64(statement, 5, initial_not_before + 86400);
if (sqlite3_step(statement) != SQLITE_DONE) goto done;
report = sqlite3_last_insert_rowid(store->database);
sqlite3_finalize(statement);
statement = NULL;
if (snprintf(report_id, sizeof(report_id), "tlsrpt-%" PRIi64
"-%" PRIi64 "@%s", day_start, report, submitter) >=
(int)sizeof(report_id) ||
!BongoTlsRptStoreBuildReport(store, report_domain, day_start,
reporting_policy, organization_name,
contact_info, report_id, &json)) goto done;
if (sqlite3_prepare_v2(store->database, UpdateReport, -1, &statement,
NULL) != SQLITE_OK) goto done;
sqlite3_bind_text(statement, 1, report_id, -1, SQLITE_TRANSIENT);
sqlite3_bind_text(statement, 2, json, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(statement, 3, report);
if (sqlite3_step(statement) != SQLITE_DONE) goto done;
sqlite3_finalize(statement);
statement = NULL;
for (index = 0; index < policy.endpoint_count; index++) {
if (sqlite3_prepare_v2(store->database, InsertDelivery, -1,
&statement, NULL) != SQLITE_OK) goto done;
sqlite3_bind_int64(statement, 1, report);
sqlite3_bind_int(statement, 2, policy.endpoints[index].type);
sqlite3_bind_text(statement, 3, policy.endpoints[index].uri, -1,
SQLITE_TRANSIENT);
sqlite3_bind_int64(statement, 4, initial_not_before);
if (sqlite3_step(statement) != SQLITE_DONE) goto done;
sqlite3_finalize(statement);
statement = NULL;
}
if (!BongoTlsRptStoreDeleteReport(store, report_domain, day_start,
reporting_policy) || !Commit(store))
goto done;
result = 1;
done:
sqlite3_finalize(statement);
if (json) MemFree(json);
if (result != 1) {
if (result == 0 && Commit(store)) return 0;
Rollback(store);
return -1;
}
return 1;
}
void
BongoTlsRptDeliveryFree(BongoTlsRptDelivery *delivery)
{
if (!delivery) return;
if (delivery->json) MemFree(delivery->json);
memset(delivery, 0, sizeof(*delivery));
}
int
BongoTlsRptStoreNextDelivery(BongoTlsRptStore *store, int64_t now,
BongoTlsRptDelivery *delivery,
int64_t *next_not_before)
{
static const char Due[] =
"SELECT d.id,d.endpoint_type,d.attempts,r.day_start,r.created_at,"
"r.expires_at,r.report_domain,r.report_id,d.endpoint_uri,r.json "
"FROM tlsrpt_deliveries d JOIN tlsrpt_reports r ON r.id=d.report "
"WHERE d.state=0 AND d.next_attempt<=? AND r.expires_at>? "
"ORDER BY d.next_attempt,d.id LIMIT 1";
static const char Next[] =
"SELECT min(d.next_attempt) FROM tlsrpt_deliveries d "
"JOIN tlsrpt_reports r ON r.id=d.report "
"WHERE d.state=0 AND r.expires_at>?";
sqlite3_stmt *statement = NULL;
const unsigned char *json;
int step;
int result = -1;
if (!store || !delivery || !next_not_before || now < 0) return -1;
memset(delivery, 0, sizeof(*delivery));
*next_not_before = 0;
if (sqlite3_prepare_v2(store->database, Due, -1, &statement, NULL) !=
SQLITE_OK) goto done;
sqlite3_bind_int64(statement, 1, now);
sqlite3_bind_int64(statement, 2, now);
step = sqlite3_step(statement);
if (step == SQLITE_ROW) {
delivery->id = sqlite3_column_int64(statement, 0);
delivery->endpoint_type = sqlite3_column_int(statement, 1);
delivery->attempts = sqlite3_column_int(statement, 2);
delivery->day_start = sqlite3_column_int64(statement, 3);
delivery->created_at = sqlite3_column_int64(statement, 4);
delivery->expires_at = sqlite3_column_int64(statement, 5);
json = sqlite3_column_text(statement, 9);
if (!CopyColumn(statement, 6, delivery->report_domain,
sizeof(delivery->report_domain)) ||
!CopyColumn(statement, 7, delivery->report_id,
sizeof(delivery->report_id)) ||
!CopyColumn(statement, 8, delivery->endpoint_uri,
sizeof(delivery->endpoint_uri)) || !json ||
!(delivery->json = MemStrdup((const char *)json))) goto done;
result = 1;
goto done;
}
sqlite3_finalize(statement);
statement = NULL;
if (step != SQLITE_DONE ||
sqlite3_prepare_v2(store->database, Next, -1, &statement, NULL) !=
SQLITE_OK) goto done;
sqlite3_bind_int64(statement, 1, now);
step = sqlite3_step(statement);
if (step != SQLITE_ROW) goto done;
if (sqlite3_column_type(statement, 0) != SQLITE_NULL)
*next_not_before = sqlite3_column_int64(statement, 0);
result = 0;
done:
sqlite3_finalize(statement);
if (result < 0) BongoTlsRptDeliveryFree(delivery);
return result;
}
int
BongoTlsRptStoreFinishDelivery(BongoTlsRptStore *store,
int64_t delivery_id,
BongoTlsRptDeliveryResult result,
int64_t next_attempt,
const char *error)
{
static const char Update[] =
"UPDATE tlsrpt_deliveries SET state=?,attempts=attempts+1,"
"next_attempt=?,last_error=? WHERE id=?";
static const char Finish[] =
"DELETE FROM tlsrpt_reports WHERE id=(SELECT report FROM "
"tlsrpt_deliveries WHERE id=?) AND NOT EXISTS (SELECT 1 FROM "
"tlsrpt_deliveries WHERE report=(SELECT report FROM "
"tlsrpt_deliveries WHERE id=?) AND state=0)";
sqlite3_stmt *statement = NULL;
int changed;
int state;
int ok = 0;
if (!store || delivery_id <= 0 || result < BONGO_TLSRPT_DELIVERY_RETRY ||
result > BONGO_TLSRPT_DELIVERY_PERMANENT_FAILURE ||
next_attempt < 0 || !error || strlen(error) > 4096 ||
!BeginImmediate(store)) return 0;
state = result == BONGO_TLSRPT_DELIVERY_RETRY ? 0 : (int)result;
if (sqlite3_prepare_v2(store->database, Update, -1, &statement, NULL) !=
SQLITE_OK) goto done;
sqlite3_bind_int(statement, 1, state);
sqlite3_bind_int64(statement, 2,
state == 0 ? next_attempt : 0);
sqlite3_bind_text(statement, 3, error, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(statement, 4, delivery_id);
if (sqlite3_step(statement) != SQLITE_DONE) goto done;
changed = sqlite3_changes(store->database);
sqlite3_finalize(statement);
statement = NULL;
if (changed != 1) goto done;
if (state != 0) {
if (sqlite3_prepare_v2(store->database, Finish, -1, &statement,
NULL) != SQLITE_OK) goto done;
sqlite3_bind_int64(statement, 1, delivery_id);
sqlite3_bind_int64(statement, 2, delivery_id);
if (sqlite3_step(statement) != SQLITE_DONE) goto done;
}
if (!Commit(store)) goto done;
ok = 1;
done:
sqlite3_finalize(statement);
if (!ok) Rollback(store);
return ok;
}
int
BongoTlsRptStoreCleanupExpired(BongoTlsRptStore *store, int64_t now)
{
sqlite3_stmt *statement = NULL;
int result;
if (!store || now < 0 ||
sqlite3_prepare_v2(store->database,
"DELETE FROM tlsrpt_reports WHERE expires_at<=?", -1,
&statement, NULL) != SQLITE_OK) return 0;
sqlite3_bind_int64(statement, 1, now);
result = sqlite3_step(statement) == SQLITE_DONE;
sqlite3_finalize(statement);
return result;
}