tom/hermes-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed AutoSavedStore's deletion of data when table was using composite keys.

Browse Source
This commit is contained in:
Tom
2025-01-09 03:38:23 +00:00
parent 3429c8f8dc
commit 467c3cf0b0
15 changed files with 202 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Store/ActionStore.cs
View File

@@ -5,7 +5,7 @@ using HermesSocketServer.Store.Internal;
namespace HermesSocketServer.Store
{
public class ActionStore : AutoSavedStore<string, RedeemableAction>
public class ActionStore : ComplexAutoSavedStore<string, RedeemableAction>
{
private readonly string _userId;
private readonly Database _database;
@@ -45,9 +45,5 @@ namespace HermesSocketServer.Store
protected override void OnInitialModify(string key, RedeemableAction value)
{
}
protected override void OnInitialRemove(string key)
{
}
}
}

6
Store/ChatterStore.cs
View File

@@ -4,7 +4,7 @@ using HermesSocketServer.Store.Internal;
namespace HermesSocketServer.Store
{
public class ChatterStore : AutoSavedStore<string, ChatterVoice>
public class ChatterStore : ComplexAutoSavedStore<string, ChatterVoice>
{
private readonly string _userId;
private readonly Database _database;
@@ -43,9 +43,5 @@ namespace HermesSocketServer.Store
protected override void OnInitialModify(string key, ChatterVoice value)
{
}
protected override void OnInitialRemove(string key)
{
}
}
}

39
Store/Internal/SelfGeneratedStore.cs → Store/Internal/AutoSavedStore.cs
View File

@@ -14,9 +14,12 @@ namespace HermesSocketServer.Store.Internal
public AutoSavedStore(DatabaseTable table, Database database, Serilog.ILogger logger)
: base(logger)
: base()
{
_generator = new GroupSaveSqlGenerator<V>(table.PropertyMapping, logger);
if (table.TypeMapping == null)
_generator = new GroupSaveSqlGenerator<V>(table.PropertyMapping, logger);
else
_generator = new GroupSaveSqlGenerator<V>(table.PropertyMapping, table.TypeMapping, logger);
_table = table;
_database = database;
_logger = logger;
@@ -25,21 +28,22 @@ namespace HermesSocketServer.Store.Internal
public override async Task Save()
{
var allColumns = _table.KeyColumns.Union(_table.DataColumns).ToArray();
var typeMapping = _table.TypeMapping ?? new Dictionary<string, string>();
await GenerateQuery(_added,
(size) => _generator.GeneratePreparedInsertSql(_table.TableName, size, allColumns),
async (query, _, values) => await _generator.DoPreparedStatement(_database, query, values, allColumns));
await GenerateQuery(_modified,
(size) => _generator.GeneratePreparedUpdateSql(_table.TableName, size, _table.KeyColumns, _table.DataColumns),
await GenerateQuery(_added,
(size) => _generator.GeneratePreparedInsertSql(_table.TableName, size, allColumns, typeMapping),
async (query, _, values) => await _generator.DoPreparedStatement(_database, query, values, allColumns));
await GenerateQuery(_deleted,
(size) => _generator.GeneratePreparedDeleteSql(_table.TableName, size, _table.KeyColumns),
await GenerateQuery(_modified,
(size) => _generator.GeneratePreparedUpdateSql(_table.TableName, size, _table.KeyColumns, _table.DataColumns, typeMapping),
async (query, _, values) => await _generator.DoPreparedStatement(_database, query, values, allColumns));
await GenerateQuery(_deleted,
(size) => _generator.GeneratePreparedDeleteSql(_table.TableName, size, _table.KeyColumns, typeMapping),
async (query, keys, _) => await _generator.DoPreparedStatementRaw(_database, query, keys, _table.KeyColumns));
}
private async Task GenerateQuery(IList<K> keys, Func<int, string> generate, Func<string, IEnumerable<K>, IEnumerable<V>, Task> execute)
private async Task GenerateQuery(IList<K> keys, Func<int, string> generate, Func<string, IEnumerable<K>, IEnumerable<V>, Task<int>> execute)
{
ImmutableList<K>? list = null;
lock (_lock)
@@ -52,10 +56,17 @@ namespace HermesSocketServer.Store.Internal
}
var query = generate(list.Count);
_logger.Debug($"{_table.TableName} - Adding {list.Count} rows to database: {query}");
var values = list.Select(id => _store[id]).Where(v => v != null);
await execute(query, list, values);
int rowsAffected = await execute(query, list, values);
if (rowsAffected != list.Count)
{
_logger.Error($"Rows affected in database ({rowsAffected}) and the number of items that should be modified ({list.Count}) do not match: {query}");
}
else
{
_logger.Information($"{rowsAffected} rows were affected by this query: {query}");
}
}
}
}

106
Store/Internal/ComplexAutoSavedStore.cs Normal file
View File

@@ -0,0 +1,106 @@
using System.Collections.Immutable;
using HermesSocketLibrary.db;
namespace HermesSocketServer.Store.Internal
{
public abstract class ComplexAutoSavedStore<K, V> : GroupSaveStore<K, V> where K : class where V : class
{
private readonly IList<V> _removedValues;
private readonly GroupSaveSqlGenerator<V> _generator;
private readonly DatabaseTable _table;
private readonly Database _database;
private readonly Serilog.ILogger _logger;
public ComplexAutoSavedStore(DatabaseTable table, Database database, Serilog.ILogger logger)
: base()
{
if (table.KeyColumns.Length <= 1)
throw new InvalidOperationException($"Use AutoSavedStore instead due to a single key column [table name: {table.TableName}]");
_removedValues = new List<V>();
if (table.TypeMapping == null)
_generator = new GroupSaveSqlGenerator<V>(table.PropertyMapping, logger);
else
_generator = new GroupSaveSqlGenerator<V>(table.PropertyMapping, table.TypeMapping, logger);
_table = table;
_database = database;
_logger = logger;
}
public override async Task Save()
{
var allColumns = _table.KeyColumns.Union(_table.DataColumns).ToArray();
var typeMapping = _table.TypeMapping ?? new Dictionary<string, string>();
await GenerateQuery(_added,
(size) => _generator.GeneratePreparedInsertSql(_table.TableName, size, allColumns, typeMapping),
async (query, _, values) => await _generator.DoPreparedStatement(_database, query, values, allColumns));
await GenerateQuery(_modified,
(size) => _generator.GeneratePreparedUpdateSql(_table.TableName, size, _table.KeyColumns, _table.DataColumns, typeMapping),
async (query, _, values) => await _generator.DoPreparedStatement(_database, query, values, allColumns));
await GenerateDeleteQuery(_deleted, _removedValues,
(size) => _generator.GeneratePreparedDeleteSql(_table.TableName, size, _table.KeyColumns, typeMapping),
async (query, list) => await _generator.DoPreparedStatement(_database, query, list, _table.KeyColumns));
}
private async Task GenerateQuery(IList<K> keys, Func<int, string> generate, Func<string, IEnumerable<K>, IEnumerable<V>, Task<int>> execute)
{
ImmutableList<K>? list = null;
lock (_lock)
{
if (!keys.Any())
return;
list = keys.ToImmutableList();
keys.Clear();
}
var query = generate(list.Count);
var values = list.Select(id => _store[id]).Where(v => v != null);
int rowsAffected = await execute(query, list, values);
if (rowsAffected != list.Count)
{
_logger.Error($"Rows affected in database ({rowsAffected}) and the number of items that should be modified ({list.Count}) do not match: {query}");
}
else
{
_logger.Information($"{rowsAffected} rows were affected by this query: {query}");
}
}
private async Task GenerateDeleteQuery(IList<K> keys, IList<V> values, Func<int, string> generate, Func<string, IEnumerable<V>, Task<int>> execute)
{
ImmutableList<V>? list = null;
lock (_lock)
{
if (!keys.Any() || !values.Any()) {
return;
}
list = values.ToImmutableList();
values.Clear();
keys.Clear();
}
var query = generate(list.Count);
int rowsAffected = await execute(query, list);
if (rowsAffected != list.Count)
{
_logger.Error($"Rows affected in database ({rowsAffected}) and the number of items that should be modified ({list.Count}) do not match: {query}");
}
else
{
_logger.Information($"{rowsAffected} rows were affected by this query: {query}");
}
}
protected override void OnPostRemove(K key, V value) => _removedValues.Add(value);
}
}

190
Store/Internal/GroupSaveSqlGenerator.cs
View File

@@ -2,22 +2,21 @@ using System.Reflection;
using System.Text;
using System.Text.Json;
using HermesSocketLibrary.db;
using NpgsqlTypes;
namespace HermesSocketServer.Store.Internal
{
public class GroupSaveSqlGenerator<T>
{
private readonly IDictionary<string, PropertyInfo?> _columnPropertyRelations;
private readonly IDictionary<string, NpgsqlDbType> _columnTypes;
private readonly IDictionary<string, string> _columnTypes;
private readonly Serilog.ILogger _logger;
public GroupSaveSqlGenerator(IDictionary<string, string> columnsToProperties, Serilog.ILogger logger)
: this(columnsToProperties, new Dictionary<string, NpgsqlDbType>(), logger)
: this(columnsToProperties, new Dictionary<string, string>(), logger)
{
}
public GroupSaveSqlGenerator(IDictionary<string, string> columnsToProperties, IDictionary<string, NpgsqlDbType> columnTypes, Serilog.ILogger logger)
public GroupSaveSqlGenerator(IDictionary<string, string> columnsToProperties, IDictionary<string, string> columnTypes, Serilog.ILogger logger)
{
_columnPropertyRelations = columnsToProperties.ToDictionary(p => p.Key, p => typeof(T).GetProperty(p.Value));
_columnTypes = columnTypes;
@@ -29,11 +28,11 @@ namespace HermesSocketServer.Store.Internal
throw new ArgumentException("Some properties do not exist on the values given: " + string.Join(", ", nullProperties));
}
public async Task DoPreparedStatement<V>(Database database, string sql, IEnumerable<V> values, string[] columns)
public async Task<int> DoPreparedStatement<V>(Database database, string sql, IEnumerable<V> values, string[] columns)
{
try
{
await database.Execute(sql, (c) =>
return await database.Execute(sql, (c) =>
{
var valueCounter = 0;
foreach (var value in values)
@@ -43,12 +42,10 @@ namespace HermesSocketServer.Store.Internal
var propValue = _columnPropertyRelations[column]!.GetValue(value);
if (_columnTypes.Any() && _columnTypes.TryGetValue(column, out var type))
{
if (type == NpgsqlDbType.Jsonb)
if (type == "jsonb")
propValue = JsonSerializer.Serialize(propValue);
c.Parameters.AddWithValue(column.ToLower() + valueCounter, type, propValue ?? DBNull.Value);
}
else
c.Parameters.AddWithValue(column.ToLower() + valueCounter, propValue ?? DBNull.Value);
c.Parameters.AddWithValue(column.ToLower() + valueCounter, propValue ?? DBNull.Value);
}
valueCounter++;
}
@@ -57,14 +54,15 @@ namespace HermesSocketServer.Store.Internal
catch (Exception ex)
{
_logger.Error(ex, "Failed to execute a prepared statement: " + sql);
return -1;
}
}
public async Task DoPreparedStatementRaw<V>(Database database, string sql, IEnumerable<V> values, string[] columns)
public async Task<int> DoPreparedStatementRaw<V>(Database database, string sql, IEnumerable<V> values, string[] columns)
{
try
{
await database.Execute(sql, (c) =>
return await database.Execute(sql, (c) =>
{
var valueCounter = 0;
foreach (var value in values)
@@ -81,44 +79,11 @@ namespace HermesSocketServer.Store.Internal
catch (Exception ex)
{
_logger.Error(ex, "Failed to execute a prepared statement: " + sql);
return -1;
}
}
public string GenerateInsertSql(string table, IEnumerable<T> values, IEnumerable<string> columns)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
if (values == null)
throw new ArgumentNullException(nameof(values));
if (!values.Any())
throw new ArgumentException("Empty list given.", nameof(values));
if (columns == null)
throw new ArgumentNullException(nameof(columns));
if (!columns.Any())
throw new ArgumentException("Empty list given.", nameof(columns));
var ctp = columns.ToDictionary(c => c, c => _columnPropertyRelations[c]);
var sb = new StringBuilder();
sb.Append($"INSERT INTO \"{table}\" (\"{string.Join("\", \"", columns)}\") VALUES ");
foreach (var value in values)
{
sb.Append("(");
foreach (var column in columns)
{
var propValue = _columnPropertyRelations[column]!.GetValue(value);
var propType = _columnPropertyRelations[column]!.PropertyType;
WriteValue(sb, propValue ?? DBNull.Value, propType);
sb.Append(",");
}
sb.Remove(sb.Length - 1, 1)
.Append("),");
}
sb.Remove(sb.Length - 1, 1)
.Append(';');
return sb.ToString();
}
public string GeneratePreparedInsertSql(string table, int rows, IEnumerable<string> columns)
public string GeneratePreparedInsertSql(string table, int rows, IEnumerable<string> columns, IDictionary<string, string> typeMapping)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
@@ -138,8 +103,14 @@ namespace HermesSocketServer.Store.Internal
{
sb.Append('@')
.Append(column)
.Append(row)
.Append(", ");
.Append(row);
if (typeMapping.TryGetValue(column, out var type))
sb.Append("::\"")
.Append(type)
.Append("\"");
sb.Append(", ");
}
sb.Remove(sb.Length - 2, 2)
.Append("),");
@@ -149,49 +120,7 @@ namespace HermesSocketServer.Store.Internal
return sb.ToString();
}
public string GenerateUpdateSql(string table, IEnumerable<T> values, IEnumerable<string> keyColumns, IEnumerable<string> updateColumns)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
if (values == null)
throw new ArgumentNullException(nameof(values));
if (!values.Any())
throw new ArgumentException("Empty list given.", nameof(values));
if (keyColumns == null)
throw new ArgumentNullException(nameof(keyColumns));
if (!keyColumns.Any())
throw new ArgumentException("Empty list given.", nameof(keyColumns));
if (updateColumns == null)
throw new ArgumentNullException(nameof(updateColumns));
if (!updateColumns.Any())
throw new ArgumentException("Empty list given.", nameof(updateColumns));
var columns = keyColumns.Union(updateColumns);
var ctp = columns.ToDictionary(c => c, c => _columnPropertyRelations[c]);
var sb = new StringBuilder();
sb.Append($"UPDATE \"{table}\" as t SET {string.Join(", ", updateColumns.Select(c => "\"" + c + "\" = c.\"" + c + "\""))} FROM (VALUES ");
foreach (var value in values)
{
sb.Append("(");
foreach (var column in columns)
{
var propValue = _columnPropertyRelations[column]!.GetValue(value);
var propType = _columnPropertyRelations[column]!.PropertyType;
WriteValue(sb, propValue, propType);
sb.Append(",");
}
sb.Remove(sb.Length - 1, 1)
.Append("),");
}
sb.Remove(sb.Length - 1, 1)
.Append($") AS c(\"{string.Join("\", \"", columns)}\") WHERE ")
.Append(string.Join(" AND ", keyColumns.Select(c => "t.\"" + c + "\" = c.\"" + c + "\"")))
.Append(";");
return sb.ToString();
}
public string GeneratePreparedUpdateSql(string table, int rows, IEnumerable<string> keyColumns, IEnumerable<string> updateColumns)
public string GeneratePreparedUpdateSql(string table, int rows, IEnumerable<string> keyColumns, IEnumerable<string> updateColumns, IDictionary<string, string> typeMapping)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
@@ -215,8 +144,14 @@ namespace HermesSocketServer.Store.Internal
{
sb.Append('@')
.Append(column)
.Append(row)
.Append(", ");
.Append(row);
if (typeMapping.TryGetValue(column, out var type))
sb.Append("::\"")
.Append(type)
.Append("\"");
sb.Append(", ");
}
sb.Remove(sb.Length - 2, 2)
.Append("),");
@@ -229,41 +164,7 @@ namespace HermesSocketServer.Store.Internal
return sb.ToString();
}
public string GenerateDeleteSql(string table, IEnumerable<string> keys, IEnumerable<string> keyColumns)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
if (keys == null)
throw new ArgumentNullException(nameof(keys));
if (!keys.Any())
throw new ArgumentException("Empty list given.", nameof(keys));
if (keyColumns == null)
throw new ArgumentNullException(nameof(keyColumns));
if (!keyColumns.Any())
throw new ArgumentException("Empty list given.", nameof(keyColumns));
var ctp = keyColumns.ToDictionary(c => c, c => _columnPropertyRelations[c]);
var sb = new StringBuilder();
sb.Append($"DELETE FROM \"{table}\" WHERE (\"{string.Join("\", \"", keyColumns)}\") IN (");
foreach (var k in keys)
{
sb.Append("(");
foreach (var column in keyColumns)
{
var propType = _columnPropertyRelations[column]!.PropertyType;
WriteValue(sb, k, propType);
sb.Append(",");
}
sb.Remove(sb.Length - 1, 1)
.Append("),");
}
sb.Remove(sb.Length - 1, 1)
.Append(");");
return sb.ToString();
}
public string GeneratePreparedDeleteSql(string table, int rows, IEnumerable<string> keyColumns)
public string GeneratePreparedDeleteSql(string table, int rows, IEnumerable<string> keyColumns, IDictionary<string, string> typeMapping)
{
if (string.IsNullOrWhiteSpace(table))
throw new ArgumentException("Value is either null or whitespace-filled.", nameof(table));
@@ -282,8 +183,14 @@ namespace HermesSocketServer.Store.Internal
{
sb.Append('@')
.Append(column)
.Append(row)
.Append(", ");
.Append(row);
if (typeMapping.TryGetValue(column, out var type))
sb.Append("::\"")
.Append(type)
.Append("\"");
sb.Append(", ");
}
sb.Remove(sb.Length - 2, 2)
.Append("),");
@@ -293,26 +200,5 @@ namespace HermesSocketServer.Store.Internal
return sb.ToString();
}
private void WriteValue(StringBuilder sb, object? value, Type type)
{
if (type == typeof(string))
sb.Append("'")
.Append(value)
.Append("'");
else if (type == typeof(Guid))
sb.Append("uuid('")
.Append(value?.ToString())
.Append("')");
else if (type == typeof(TimeSpan))
{
if (value == null)
sb.Append("0");
else
sb.Append(((TimeSpan)value).TotalMilliseconds);
}
else
sb.Append(value);
}
}
}

31
Store/Internal/GroupedSaveStore.cs
View File

@@ -6,7 +6,6 @@ namespace HermesSocketServer.Store.Internal
{
public abstract class GroupSaveStore<K, V> : IStore<K, V> where K : class where V : class
{
private readonly Serilog.ILogger _logger;
protected readonly IDictionary<K, V> _store;
protected readonly IList<K> _added;
protected readonly IList<K> _modified;
@@ -14,9 +13,8 @@ namespace HermesSocketServer.Store.Internal
protected readonly object _lock;
public GroupSaveStore(Serilog.ILogger logger)
public GroupSaveStore()
{
_logger = logger;
_store = new Dictionary<K, V>();
_added = new List<K>();
_modified = new List<K>();
@@ -27,7 +25,7 @@ namespace HermesSocketServer.Store.Internal
public abstract Task Load();
protected abstract void OnInitialAdd(K key, V value);
protected abstract void OnInitialModify(K key, V value);
protected abstract void OnInitialRemove(K key);
protected abstract void OnPostRemove(K key, V value);
public abstract Task Save();
public V? Get(K key)
@@ -52,7 +50,7 @@ namespace HermesSocketServer.Store.Internal
{
if (key == null)
return false;
lock (_lock)
{
if (_store.TryGetValue(key, out V? value))
@@ -65,7 +63,6 @@ namespace HermesSocketServer.Store.Internal
if (!_added.Contains(key) && !_modified.Contains(key))
{
_modified.Add(key);
_logger.Information($"added key to _modified {key}");
}
return true;
}
@@ -80,21 +77,21 @@ namespace HermesSocketServer.Store.Internal
lock (_lock)
{
OnInitialRemove(key);
if (_store.Remove(key))
if (_store.TryGetValue(key, out var value))
{
_logger.Information($"removed key from _deleted {key}");
if (!_added.Remove(key))
if (_store.Remove(key))
{
_modified.Remove(key);
_logger.Information($"removed key from _added & _modified {key}");
if (!_deleted.Contains(key))
OnPostRemove(key, value);
if (!_added.Remove(key))
{
_deleted.Add(key);
_logger.Information($"added key to _deleted {key}");
_modified.Remove(key);
if (!_deleted.Contains(key))
{
_deleted.Add(key);
}
}
return true;
}
return true;
}
}
return false;
@@ -116,7 +113,6 @@ namespace HermesSocketServer.Store.Internal
if (!_added.Contains(key) && !_modified.Contains(key))
{
_modified.Add(key);
_logger.Information($"added key to _modified {key}");
}
return true;
}
@@ -128,7 +124,6 @@ namespace HermesSocketServer.Store.Internal
if (!_deleted.Remove(key) && !_added.Contains(key))
{
_added.Add(key);
_logger.Information($"added key to _added {key}");
}
return true;
}

12
Store/PolicyStore.cs
View File

@@ -1,10 +1,10 @@
using HermesSocketLibrary.db;
using HermesSocketServer.Models;
using HermesSocketServer.Messages;
using HermesSocketServer.Store.Internal;
namespace HermesSocketServer.Store
{
public class PolicyStore : AutoSavedStore<string, PolicyMessage>
public class PolicyStore : AutoSavedStore<string, Policy>
{
private readonly string _userId;
private readonly Database _database;
@@ -26,7 +26,7 @@ namespace HermesSocketServer.Store
await _database.Execute(sql, data, (reader) =>
{
var id = reader.GetGuid(0);
_store.Add(id.ToString(), new PolicyMessage()
_store.Add(id.ToString(), new Policy()
{
Id = id,
UserId = _userId,
@@ -39,15 +39,15 @@ namespace HermesSocketServer.Store
_logger.Information($"Loaded {_store.Count} policies from database.");
}
protected override void OnInitialAdd(string key, PolicyMessage value)
protected override void OnInitialAdd(string key, Policy value)
{
}
protected override void OnInitialModify(string key, PolicyMessage value)
protected override void OnInitialModify(string key, Policy value)
{
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, Policy value)
{
}
}

2
Store/RedemptionStore.cs
View File

@@ -47,7 +47,7 @@ namespace HermesSocketServer.Store
{
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, Redemption value)
{
}
}

2
Store/TTSFilterStore.cs
View File

@@ -44,7 +44,7 @@ namespace HermesSocketServer.Store
{
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, TTSWordFilter value)
{
}
}

2
Store/UserStore.cs
View File

@@ -43,7 +43,7 @@ namespace HermesSocketServer.Store
{
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, User value)
{
}
}

4
Store/VoiceStateStore.cs
View File

@@ -5,7 +5,7 @@ using HermesSocketServer.Validators;
namespace HermesSocketServer.Store
{
public class VoiceStateStore : AutoSavedStore<string, TTSVoiceState>
public class VoiceStateStore : ComplexAutoSavedStore<string, TTSVoiceState>
{
private readonly string _userId;
private readonly VoiceIdValidator _idValidator;
@@ -48,7 +48,7 @@ namespace HermesSocketServer.Store
{
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, TTSVoiceState value)
{
}
}

2
Store/VoiceStore.cs
View File

@@ -48,7 +48,7 @@ namespace HermesSocketServer.Store
_nameValidator.Check(value.Name);
}
protected override void OnInitialRemove(string key)
protected override void OnPostRemove(string key, TTSVoice value)
{
}
}