Add UnrealDB reading capability to reputation module.

We still write the old format, unless encryption is enabled.
This commit is contained in:
Bram Matthys 2021-05-03 15:14:56 +02:00
parent dde3e0ccb2
commit 40ace27e7d
No known key found for this signature in database
GPG key ID: BF8116B163EAAE98

View file

@ -19,9 +19,12 @@
#define REPUTATION_VERSION "1.2"
/* Change to #define to benchmark. Note that this will add random
* reputation entries so should never be used on production servers!!!
*/
#undef BENCHMARK
#undef TEST
#undef BENCHMARK
/* Benchmark results (2GHz Xeon Skylake, compiled with -O2, Linux):
* 10k random IP's with various expire times:
* - load db: 23 ms
@ -76,12 +79,30 @@ ModuleHeader MOD_HEADER
#define Reputation(client) moddata_client(client, reputation_md).l
#define WARN_WRITE_ERROR(fname) \
do { \
sendto_realops_and_log("[reputation] Error writing to temporary database file " \
"'%s': %s (DATABASE NOT SAVED)", \
fname, unrealdb_get_error_string()); \
} while(0)
#define W_SAFE(x) \
do { \
if (!(x)) { \
WARN_WRITE_ERROR(tmpfname); \
unrealdb_close(db); \
return 0; \
} \
} while(0)
/* Definitions (structs, etc.) */
struct cfgstruct {
int expire_score[MAXEXPIRES];
long expire_time[MAXEXPIRES];
char *database;
char *db_secret;
};
typedef struct ReputationEntry ReputationEntry;
@ -97,6 +118,7 @@ struct ReputationEntry {
/* Global variables */
static struct cfgstruct cfg; /**< Current configuration */
static struct cfgstruct test; /**< Testing configuration (not active yet) */
long reputation_starttime = 0;
long reputation_writtentime = 0;
@ -111,7 +133,8 @@ ModDataInfo *reputation_md; /* Module Data structure which we acquire */
void reputation_md_free(ModData *m);
char *reputation_md_serialize(ModData *m);
void reputation_md_unserialize(char *str, ModData *m);
void config_setdefaults(void);
void reputation_config_setdefaults(struct cfgstruct *cfg);
void reputation_free_config(struct cfgstruct *cfg);
CMD_FUNC(reputation_cmd);
CMD_FUNC(reputationunperm);
int reputation_whois(Client *client, Client *target);
@ -122,18 +145,21 @@ int reputation_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
int reputation_config_run(ConfigFile *cf, ConfigEntry *ce, int type);
int reputation_config_posttest(int *errs);
static uint64_t hash_reputation_entry(char *ip);
ReputationEntry *find_reputation_entry(char *ip);
void add_reputation_entry(ReputationEntry *e);
EVENT(delete_old_records);
EVENT(add_scores);
EVENT(save_db_evt);
void load_db(void);
void save_db(void);
EVENT(reputation_save_db_evt);
int reputation_load_db(void);
int reputation_save_db(void);
int reputation_starttime_callback(void);
MOD_TEST()
{
memcpy(&ModInf, modinfo, modinfo->size);
memset(&cfg, 0, sizeof(cfg));
memset(&test, 0, sizeof(cfg));
reputation_config_setdefaults(&test);
HookAdd(modinfo->handle, HOOKTYPE_CONFIGTEST, 0, reputation_config_test);
HookAdd(modinfo->handle, HOOKTYPE_CONFIGPOSTTEST, 0, reputation_config_posttest);
CallbackAddEx(modinfo->handle, CALLBACKTYPE_REPUTATION_STARTTIME, reputation_starttime_callback);
@ -160,7 +186,7 @@ MOD_INIT()
if (!reputation_md)
abort();
config_setdefaults();
reputation_config_setdefaults(&cfg);
HookAdd(modinfo->handle, HOOKTYPE_CONFIGRUN, 0, reputation_config_run);
HookAdd(modinfo->handle, HOOKTYPE_WHOIS, 0, reputation_whois);
HookAdd(modinfo->handle, HOOKTYPE_HANDSHAKE, 0, reputation_set_on_connect);
@ -172,43 +198,78 @@ MOD_INIT()
return MOD_SUCCESS;
}
#ifdef BENCHMARK
void reputation_benchmark(int entries)
{
char ip[64];
int i;
ReputationEntry *e;
srand(1234); // fixed seed
for (i = 0; i < entries; i++)
{
ReputationEntry *e = safe_alloc(sizeof(ReputationEntry) + 64);
snprintf(e->ip, 63, "%d.%d.%d.%d", rand()%255, rand()%255, rand()%255, rand()%255);
e->score = rand()%255 + 1;
e->last_seen = TStime();
if (find_reputation_entry(e->ip))
{
safe_free(e);
continue;
}
add_reputation_entry(e);
}
}
#endif
MOD_LOAD()
{
load_db();
reputation_load_db();
if (reputation_starttime == 0)
reputation_starttime = TStime();
EventAdd(ModInf.handle, "delete_old_records", delete_old_records, NULL, DELETE_OLD_EVERY*1000, 0);
EventAdd(ModInf.handle, "add_scores", add_scores, NULL, BUMP_SCORE_EVERY*1000, 0);
EventAdd(ModInf.handle, "save_db", save_db_evt, NULL, SAVE_DB_EVERY*1000, 0);
EventAdd(ModInf.handle, "reputation_save_db", reputation_save_db_evt, NULL, SAVE_DB_EVERY*1000, 0);
#ifdef BENCHMARK
reputation_benchmark(10000);
#endif
return MOD_SUCCESS;
}
MOD_UNLOAD()
{
save_db();
reputation_save_db();
reputation_free_config(&test);
reputation_free_config(&cfg);
return MOD_SUCCESS;
}
void config_setdefaults(void)
void reputation_config_setdefaults(struct cfgstruct *cfg)
{
/* data/reputation.db */
safe_strdup(cfg.database, "reputation.db");
convert_to_absolute_path(&cfg.database, PERMDATADIR);
safe_strdup(cfg->database, "reputation.db");
convert_to_absolute_path(&cfg->database, PERMDATADIR);
/* EXPIRES the following entries if the IP does appear for some time: */
/* <=2 points after 1 hour */
cfg.expire_score[0] = 2;
cfg->expire_score[0] = 2;
#ifndef TEST
cfg.expire_time[0] = 3600;
cfg->expire_time[0] = 3600;
#else
cfg.expire_time[0] = 36;
cfg->expire_time[0] = 36;
#endif
/* <=6 points after 7 days */
cfg.expire_score[1] = 6;
cfg.expire_time[1] = 86400*7;
cfg->expire_score[1] = 6;
cfg->expire_time[1] = 86400*7;
/* ANY result that has not been seen for 30 days */
cfg.expire_score[2] = -1;
cfg.expire_time[2] = 86400*30;
cfg->expire_score[2] = -1;
cfg->expire_time[2] = 86400*30;
}
void reputation_free_config(struct cfgstruct *cfg)
{
safe_free(cfg->database);
safe_free(cfg->db_secret);
}
int reputation_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
@ -218,11 +279,11 @@ int reputation_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
if (type != CONFIG_SET)
return 0;
/* We are only interrested in set::reputation.. */
if (!ce || strcmp(ce->ce_varname, "reputation"))
return 0;
for (cep = ce->ce_entries; cep; cep = cep->ce_next)
{
if (!cep->ce_vardata)
@ -235,6 +296,18 @@ int reputation_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
if (!strcmp(cep->ce_varname, "database"))
{
convert_to_absolute_path(&cep->ce_vardata, PERMDATADIR);
safe_strdup(test.database, cep->ce_vardata);
} else
if (!strcmp(cep->ce_varname, "db-secret"))
{
char *err;
if ((err = unrealdb_test_secret(cep->ce_vardata)))
{
config_error("%s:%i: set::channeldb::db-secret: %s", cep->ce_fileptr->cf_filename, cep->ce_varlinenum, err);
errors++;
continue;
}
safe_strdup(test.db_secret, cep->ce_vardata);
} else
{
config_error("%s:%i: unknown directive set::reputation::%s",
@ -243,7 +316,7 @@ int reputation_config_test(ConfigFile *cf, ConfigEntry *ce, int type, int *errs)
continue;
}
}
*errs = errors;
return errors ? -1 : 1;
}
@ -254,16 +327,20 @@ int reputation_config_run(ConfigFile *cf, ConfigEntry *ce, int type)
if (type != CONFIG_SET)
return 0;
/* We are only interrested in set::reputation.. */
if (!ce || strcmp(ce->ce_varname, "reputation"))
return 0;
for (cep = ce->ce_entries; cep; cep = cep->ce_next)
{
if (!strcmp(cep->ce_varname, "database"))
{
safe_strdup(cfg.database, cep->ce_vardata);
} else
if (!strcmp(cep->ce_varname, "db-secret"))
{
safe_strdup(cfg.db_secret, cep->ce_vardata);
}
}
return 1;
@ -272,13 +349,20 @@ int reputation_config_run(ConfigFile *cf, ConfigEntry *ce, int type)
int reputation_config_posttest(int *errs)
{
int errors = 0;
char *errstr;
if (test.database && ((errstr = unrealdb_test_db(test.database, test.db_secret))))
{
config_error("[reputation] %s", errstr);
errors++;
}
*errs = errors;
return errors ? -1 : 1;
}
/** Parse database header and set variables appropriately */
int parse_db_header(char *buf)
int parse_db_header_old(char *buf)
{
char *header=NULL, *version=NULL, *starttime=NULL, *writtentime=NULL;
char *p=NULL;
@ -308,7 +392,7 @@ int parse_db_header(char *buf)
return 1;
}
void load_db(void)
void reputation_load_db_old(void)
{
FILE *fd;
char buf[512], *p;
@ -324,7 +408,7 @@ void load_db(void)
config_warn("WARNING: Could not open/read database '%s': %s", cfg.database, strerror(ERRNO));
return;
}
memset(buf, 0, sizeof(buf));
if (fgets(buf, 512, fd) == NULL)
{
@ -332,7 +416,7 @@ void load_db(void)
fclose(fd);
return;
}
/* Header contains: REPDB <version> <starttime> <writtentime>
* Where:
* REPDB: Literally the string "REPDB".
@ -341,7 +425,7 @@ void load_db(void)
* in other words: when this module was first loaded, ever.
* <writtentime> Time that the database was last written.
*/
if (!parse_db_header(buf))
if (!parse_db_header_old(buf))
{
config_error("WARNING: Cannot load database %s. Error reading header. "
"Database corrupt? Or are you downgrading from a newer "
@ -355,7 +439,7 @@ void load_db(void)
{
char *ip = NULL, *score = NULL, *last_seen = NULL;
ReputationEntry *e;
stripcrlf(buf);
/* Format: <ip> <score> <last seen> */
ip = strtoken(&p, buf, " ");
@ -367,12 +451,12 @@ void load_db(void)
last_seen = strtoken(&p, NULL, " ");
if (!last_seen)
continue;
e = safe_alloc(sizeof(ReputationEntry)+strlen(ip));
strcpy(e->ip, ip); /* safe, see alloc above */
e->score = atoi(score);
e->last_seen = atol(last_seen);
add_reputation_entry(e);
}
fclose(fd);
@ -384,7 +468,129 @@ void load_db(void)
#endif
}
void save_db(void)
#define R_SAFE(x) \
do { \
if (!(x)) { \
config_warn("[reputation] Read error from database file '%s' (possible corruption): %s", cfg.database, unrealdb_get_error_string()); \
unrealdb_close(db); \
safe_free(ip); \
return 0; \
} \
} while(0)
int reputation_load_db_new(UnrealDB *db)
{
uint64_t l_db_version = 0;
uint64_t l_starttime = 0;
uint64_t l_writtentime = 0;
uint64_t count = 0;
uint64_t i;
char *ip = NULL;
uint16_t score;
uint64_t last_seen;
ReputationEntry *e;
#ifdef BENCHMARK
struct timeval tv_alpha, tv_beta;
gettimeofday(&tv_alpha, NULL);
#endif
R_SAFE(unrealdb_read_int64(db, &l_db_version)); /* reputation db version */
if (l_db_version > 2)
{
config_error("[reputation] Reputation DB is of a newer version (%ld) than supported by us (%ld). "
"Did you perhaps downgrade your UnrealIRCd?",
(long)l_db_version, (long)2);
unrealdb_close(db);
return 0;
}
R_SAFE(unrealdb_read_int64(db, &l_starttime)); /* starttime of data gathering */
R_SAFE(unrealdb_read_int64(db, &l_writtentime)); /* current time */
R_SAFE(unrealdb_read_int64(db, &count)); /* number of entries */
reputation_starttime = l_starttime;
reputation_writtentime = l_writtentime;
for (i=0; i < count; i++)
{
R_SAFE(unrealdb_read_str(db, &ip));
R_SAFE(unrealdb_read_int16(db, &score));
R_SAFE(unrealdb_read_int64(db, &last_seen));
e = safe_alloc(sizeof(ReputationEntry)+strlen(ip));
strcpy(e->ip, ip); /* safe, see alloc above */
e->score = score;
e->last_seen = last_seen;
add_reputation_entry(e);
}
unrealdb_close(db);
#ifdef BENCHMARK
gettimeofday(&tv_beta, NULL);
ircd_log(LOG_ERROR, "Reputation benchmark: LOAD DB: %lld microseconds",
(long long)(((tv_beta.tv_sec - tv_alpha.tv_sec) * 1000000) + (tv_beta.tv_usec - tv_alpha.tv_usec)));
#endif
return 1;
}
/** Load the reputation DB.
* Strategy is:
* 1) Check for the old header "REPDB 1", if so then call reputation_load_db_old().
* 2) Otherwise, open with unrealdb routine
* 3) If that fails due to a password provided but the file is unrealdb without password
* then fallback to open without a password (so users can easily upgrade to encrypted)
*/
int reputation_load_db(void)
{
FILE *fd;
UnrealDB *db;
char buf[512];
fd = fopen(cfg.database, "r");
if (!fd)
{
/* Database does not exist. Could be first boot */
config_warn("[reputation] No database present at '%s', will start a new one", cfg.database);
return 1;
}
*buf = '\0';
if (fgets(buf, sizeof(buf), fd) == NULL)
{
fclose(fd);
config_warn("[reputation] Database at '%s' is 0 bytes", cfg.database);
return 1;
}
fclose(fd);
if (!strncmp(buf, "REPDB 1 ", 8))
{
reputation_load_db_old();
return 1; /* not so good to always pretend succes */
}
/* Otherwise, it is an unrealdb, crypted or not */
db = unrealdb_open(cfg.database, UNREALDB_MODE_READ, cfg.db_secret);
if (!db)
{
if (unrealdb_get_error_code() == UNREALDB_ERROR_FILENOTFOUND)
{
/* Database does not exist. Could be first boot */
config_warn("[reputation] No database present at '%s', will start a new one", cfg.database);
return 1;
} else
if (unrealdb_get_error_code() == UNREALDB_ERROR_NOTCRYPTED)
{
db = unrealdb_open(cfg.database, UNREALDB_MODE_READ, NULL);
}
if (!db)
{
config_error("[reputation] Unable to open the database file '%s' for reading: %s", cfg.database, unrealdb_get_error_string());
return 0;
}
}
return reputation_load_db_new(db);
}
int reputation_save_db_old(void)
{
FILE *fd;
char tmpfname[512];
@ -395,19 +601,15 @@ void save_db(void)
gettimeofday(&tv_alpha, NULL);
#endif
#ifdef TEST
sendto_realops("REPUTATION IS RUNNING IN TEST MODE. SAVING DB'S...");
#endif
/* We write to a temporary file. Only to rename it later if everything was ok */
snprintf(tmpfname, sizeof(tmpfname), "%s.%x.tmp", cfg.database, getrandom32());
fd = fopen(tmpfname, "w");
if (!fd)
{
config_error("ERROR: Could not open/write database '%s': %s -- DATABASE *NOT* SAVED!!!", tmpfname, strerror(ERRNO));
return;
return 0;
}
if (fprintf(fd, "REPDB 1 %lld %lld\n", (long long)reputation_starttime, (long long)TStime()) < 0)
@ -422,7 +624,7 @@ void save_db(void)
write_fail:
config_error("ERROR writing to '%s': %s -- DATABASE *NOT* SAVED!!!", tmpfname, strerror(ERRNO));
fclose(fd);
return;
return 0;
}
}
}
@ -430,9 +632,9 @@ write_fail:
if (fclose(fd) < 0)
{
config_error("ERROR writing to '%s': %s -- DATABASE *NOT* SAVED!!!", tmpfname, strerror(ERRNO));
return;
return 0;
}
/* Everything went fine. We rename our temporary file to the existing
* DB file (will overwrite), which is more or less an atomic operation.
*/
@ -444,7 +646,7 @@ write_fail:
{
config_error("ERROR renaming '%s' to '%s': %s -- DATABASE *NOT* SAVED!!!",
tmpfname, cfg.database, strerror(ERRNO));
return;
return 0;
}
reputation_writtentime = TStime();
@ -455,7 +657,91 @@ write_fail:
(long long)(((tv_beta.tv_sec - tv_alpha.tv_sec) * 1000000) + (tv_beta.tv_usec - tv_alpha.tv_usec)));
#endif
return;
return 1;
}
int reputation_save_db(void)
{
UnrealDB *db;
char tmpfname[512];
int i;
uint64_t count;
ReputationEntry *e;
#ifdef BENCHMARK
struct timeval tv_alpha, tv_beta;
gettimeofday(&tv_alpha, NULL);
#endif
#ifdef TEST
sendto_realops("REPUTATION IS RUNNING IN TEST MODE. SAVING DB'S...");
#endif
/* Comment this out after one or more releases (means you cannot downgrade to <=5.0.9.1 anymore) */
if (cfg.db_secret == NULL)
return reputation_save_db_old();
/* We write to a temporary file. Only to rename it later if everything was ok */
snprintf(tmpfname, sizeof(tmpfname), "%s.%x.tmp", cfg.database, getrandom32());
db = unrealdb_open(tmpfname, UNREALDB_MODE_WRITE, cfg.db_secret);
if (!db)
{
WARN_WRITE_ERROR(tmpfname);
return 0;
}
/* Write header */
W_SAFE(unrealdb_write_int64(db, 2)); /* reputation db version */
W_SAFE(unrealdb_write_int64(db, reputation_starttime)); /* starttime of data gathering */
W_SAFE(unrealdb_write_int64(db, TStime())); /* current time */
/* Count entries */
count = 0;
for (i = 0; i < REPUTATION_HASH_TABLE_SIZE; i++)
for (e = ReputationHashTable[i]; e; e = e->next)
count++;
W_SAFE(unrealdb_write_int64(db, count)); /* Number of DB entries */
/* Now write the actual individual entries: */
for (i = 0; i < REPUTATION_HASH_TABLE_SIZE; i++)
{
for (e = ReputationHashTable[i]; e; e = e->next)
{
W_SAFE(unrealdb_write_str(db, e->ip));
W_SAFE(unrealdb_write_int16(db, e->score));
W_SAFE(unrealdb_write_int64(db, e->last_seen));
}
}
if (!unrealdb_close(db))
{
WARN_WRITE_ERROR(tmpfname);
return 0;
}
/* Everything went fine. We rename our temporary file to the existing
* DB file (will overwrite), which is more or less an atomic operation.
*/
#ifdef _WIN32
/* The rename operation cannot be atomic on Windows as it will cause a "file exists" error */
unlink(cfg.database);
#endif
if (rename(tmpfname, cfg.database) < 0)
{
config_error("ERROR renaming '%s' to '%s': %s -- DATABASE *NOT* SAVED!!!",
tmpfname, cfg.database, strerror(ERRNO));
return 0;
}
reputation_writtentime = TStime();
#ifdef BENCHMARK
gettimeofday(&tv_beta, NULL);
ircd_log(LOG_ERROR, "Reputation benchmark: SAVE DB: %lld microseconds",
(long long)(((tv_beta.tv_sec - tv_alpha.tv_sec) * 1000000) + (tv_beta.tv_usec - tv_alpha.tv_usec)));
#endif
return 1;
}
static uint64_t hash_reputation_entry(char *ip)
@ -548,7 +834,7 @@ EVENT(add_scores)
*/
#define MARKER_UNREGISTERED_USER (marker)
#define MARKER_REGISTERED_USER (marker+1)
list_for_each_entry(client, &client_list, client_node)
{
if (!IsUser(client))
@ -625,13 +911,13 @@ EVENT(delete_old_records)
gettimeofday(&tv_alpha, NULL);
#endif
for (i = 0; i < REPUTATION_HASH_TABLE_SIZE; i++)
{
for (e = ReputationHashTable[i]; e; e = e_next)
{
e_next = e->next;
if (is_reputation_expired(e))
{
#ifdef DEBUGMODE
@ -651,9 +937,9 @@ EVENT(delete_old_records)
#endif
}
EVENT(save_db_evt)
EVENT(reputation_save_db_evt)
{
save_db();
reputation_save_db();
}
CMD_FUNC(reputationunperm)
@ -801,7 +1087,7 @@ CMD_FUNC(reputation_user_cmd)
sendnumeric(client, ERR_NOPRIVILEGES);
return;
}
if ((parc < 2) || BadPtr(parv[1]))
{
sendnotice(client, "Reputation module statistics:");
@ -825,7 +1111,7 @@ CMD_FUNC(reputation_user_cmd)
sendnotice(client, "/REPUTATION <NN List users with reputation score below value NN");
return;
}
if (strchr(parv[1], '.') || strchr(parv[1], ':'))
{
ip = parv[1];
@ -871,7 +1157,7 @@ CMD_FUNC(reputation_user_cmd)
return;
}
}
e = find_reputation_entry(ip);
if (!e)
{
@ -927,7 +1213,7 @@ CMD_FUNC(reputation_server_cmd)
sendnumeric(client, ERR_NEEDMOREPARAMS, "REPUTATION");
return;
}
ip = parv[1];
if (parv[2][0] == '*')
@ -1003,7 +1289,7 @@ int reputation_whois(Client *client, Client *target)
if (!IsOper(client))
return 0; /* only opers can see this.. */
if (reputation > 0)
{
sendto_one(client, NULL, ":%s %d %s %s :is using an IP with a reputation score of %d",