-
Notifications
You must be signed in to change notification settings - Fork 934
Add a generic batcher for insert/update/delete statements, usable with PostgreSQL and others #1588
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 5 commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
4a2a40c
Added batching support for insert/update/delete statements on PostgreSQL
maca88 1014e35
Added an automatic flush when the max number of parameters are reache…
maca88 2375c6d
Removed unsupported clients for the generic batcher from tests
maca88 2fd9868
Fixed the max number of parameters so that we will never exceed the l…
maca88 eb10f86
Reuse parameters instead of copying them
maca88 48e7c01
Added MaxNumberOfParameters to Dialect
maca88 9ff621c
Redesigned the BatchingCommandSet for better performance
maca88 11b45e7
Reverted the changes in DriverBase and simplified BatchingCommandSet
maca88 e228b2b
Add a performance check of the generic batcher.
fredericDelaporte aca8e39
Set the generic batcher as default batcher for drivers benefiting of it.
fredericDelaporte ed97fa9
Assert corrected and used lambda for getter
maca88 651de92
Merge pull request #1 from fredericDelaporte/GenericBatcher
maca88 9232cbb
Moved statement terminator setting into dialect and done some minor r…
maca88 2b33d95
Merge branch 'master' into PostgreSQLBatcher
hazzik File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
155 changes: 155 additions & 0 deletions
155
src/NHibernate.Test/Ado/GenericBatchingBatcherFixture.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,155 @@ | ||
using System; | ||
using System.Collections; | ||
using System.Linq; | ||
using NHibernate.AdoNet; | ||
using NHibernate.Cfg; | ||
using NHibernate.Dialect; | ||
using NUnit.Framework; | ||
using Environment = NHibernate.Cfg.Environment; | ||
|
||
namespace NHibernate.Test.Ado | ||
{ | ||
[TestFixture] | ||
public class GenericBatchingBatcherFixture : TestCase | ||
{ | ||
protected override string MappingsAssembly => "NHibernate.Test"; | ||
|
||
protected override IList Mappings => new[] {"Ado.VerySimple.hbm.xml"}; | ||
|
||
protected override void Configure(Configuration configuration) | ||
{ | ||
configuration.SetProperty(Environment.BatchStrategy, typeof(GenericBatchingBatcherFactory).AssemblyQualifiedName); | ||
configuration.SetProperty(Environment.GenerateStatistics, "true"); | ||
configuration.SetProperty(Environment.BatchSize, "1000"); | ||
} | ||
|
||
protected override bool AppliesTo(Dialect.Dialect dialect) | ||
{ | ||
return !(dialect is FirebirdDialect) && | ||
!(dialect is Oracle8iDialect) && | ||
!(dialect is MsSqlCeDialect); | ||
} | ||
|
||
[Test] | ||
public void MassiveInsertUpdateDeleteTest() | ||
{ | ||
var totalRecords = 1000; | ||
BatchInsert(totalRecords); | ||
BatchUpdate(totalRecords); | ||
BatchDelete(totalRecords); | ||
|
||
DbShoudBeEmpty(); | ||
} | ||
|
||
[Test] | ||
public void BatchSizeTest() | ||
{ | ||
using (var sqlLog = new SqlLogSpy()) | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
s.SetBatchSize(5); | ||
for (var i = 0; i < 20; i++) | ||
{ | ||
s.Save(new VerySimple { Id = 1 + i, Name = $"Fabio{i}", Weight = 1.45 + i }); | ||
} | ||
tx.Commit(); | ||
|
||
var log = sqlLog.GetWholeLog(); | ||
Assert.That(4, Is.EqualTo(FindAllOccurrences(log, "Batch commands:"))); | ||
} | ||
Cleanup(); | ||
} | ||
|
||
private void BatchInsert(int totalRecords) | ||
{ | ||
Sfi.Statistics.Clear(); | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
for (var i = 0; i < totalRecords; i++) | ||
{ | ||
s.Save(new VerySimple {Id = 1 + i, Name = $"Fabio{i}", Weight = 1.45 + i}); | ||
} | ||
tx.Commit(); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
fredericDelaporte marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
public void BatchUpdate(int totalRecords) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = s.Query<VerySimple>().ToList(); | ||
Assert.That(items.Count, Is.EqualTo(totalRecords)); | ||
|
||
foreach (var item in items) | ||
{ | ||
item.Weight += 5; | ||
s.Update(item); | ||
} | ||
|
||
Sfi.Statistics.Clear(); | ||
tx.Commit(); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
} | ||
|
||
public void BatchDelete(int totalRecords) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = s.Query<VerySimple>().ToList(); | ||
Assert.That(items.Count, Is.EqualTo(totalRecords)); | ||
|
||
foreach (var item in items) | ||
{ | ||
s.Delete(item); | ||
} | ||
|
||
Sfi.Statistics.Clear(); | ||
tx.Commit(); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
} | ||
|
||
private void DbShoudBeEmpty() | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = s.Query<VerySimple>().ToList(); | ||
Assert.That(items.Count, Is.EqualTo(0)); | ||
|
||
tx.Commit(); | ||
} | ||
} | ||
|
||
private void Cleanup() | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (s.BeginTransaction()) | ||
{ | ||
s.CreateQuery("delete from VerySimple").ExecuteUpdate(); | ||
s.Transaction.Commit(); | ||
} | ||
} | ||
|
||
private int FindAllOccurrences(string source, string substring) | ||
{ | ||
if (source == null) | ||
{ | ||
return 0; | ||
} | ||
int n = 0, count = 0; | ||
while ((n = source.IndexOf(substring, n, StringComparison.InvariantCulture)) != -1) | ||
{ | ||
n += substring.Length; | ||
++count; | ||
} | ||
return count; | ||
} | ||
} | ||
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
168 changes: 168 additions & 0 deletions
168
src/NHibernate.Test/Async/Ado/GenericBatchingBatcherFixture.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,168 @@ | ||
//------------------------------------------------------------------------------ | ||
// <auto-generated> | ||
// This code was generated by AsyncGenerator. | ||
// | ||
// Changes to this file may cause incorrect behavior and will be lost if | ||
// the code is regenerated. | ||
// </auto-generated> | ||
//------------------------------------------------------------------------------ | ||
|
||
|
||
using System; | ||
using System.Collections; | ||
using System.Linq; | ||
using NHibernate.AdoNet; | ||
using NHibernate.Cfg; | ||
using NHibernate.Dialect; | ||
using NUnit.Framework; | ||
using Environment = NHibernate.Cfg.Environment; | ||
using NHibernate.Linq; | ||
|
||
namespace NHibernate.Test.Ado | ||
{ | ||
using System.Threading.Tasks; | ||
using System.Threading; | ||
[TestFixture] | ||
public class GenericBatchingBatcherFixtureAsync : TestCase | ||
{ | ||
protected override string MappingsAssembly => "NHibernate.Test"; | ||
|
||
protected override IList Mappings => new[] {"Ado.VerySimple.hbm.xml"}; | ||
|
||
protected override void Configure(Configuration configuration) | ||
{ | ||
configuration.SetProperty(Environment.BatchStrategy, typeof(GenericBatchingBatcherFactory).AssemblyQualifiedName); | ||
configuration.SetProperty(Environment.GenerateStatistics, "true"); | ||
configuration.SetProperty(Environment.BatchSize, "1000"); | ||
} | ||
|
||
protected override bool AppliesTo(Dialect.Dialect dialect) | ||
{ | ||
return !(dialect is FirebirdDialect) && | ||
!(dialect is Oracle8iDialect) && | ||
!(dialect is MsSqlCeDialect); | ||
} | ||
|
||
[Test] | ||
public async Task MassiveInsertUpdateDeleteTestAsync() | ||
{ | ||
var totalRecords = 1000; | ||
await (BatchInsertAsync(totalRecords)); | ||
await (BatchUpdateAsync(totalRecords)); | ||
await (BatchDeleteAsync(totalRecords)); | ||
|
||
await (DbShoudBeEmptyAsync()); | ||
} | ||
|
||
[Test] | ||
public async Task BatchSizeTestAsync() | ||
{ | ||
using (var sqlLog = new SqlLogSpy()) | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
s.SetBatchSize(5); | ||
for (var i = 0; i < 20; i++) | ||
{ | ||
await (s.SaveAsync(new VerySimple { Id = 1 + i, Name = $"Fabio{i}", Weight = 1.45 + i })); | ||
} | ||
await (tx.CommitAsync()); | ||
|
||
var log = sqlLog.GetWholeLog(); | ||
Assert.That(4, Is.EqualTo(FindAllOccurrences(log, "Batch commands:"))); | ||
} | ||
await (CleanupAsync()); | ||
} | ||
|
||
private async Task BatchInsertAsync(int totalRecords, CancellationToken cancellationToken = default(CancellationToken)) | ||
{ | ||
Sfi.Statistics.Clear(); | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
for (var i = 0; i < totalRecords; i++) | ||
{ | ||
await (s.SaveAsync(new VerySimple {Id = 1 + i, Name = $"Fabio{i}", Weight = 1.45 + i}, cancellationToken)); | ||
} | ||
await (tx.CommitAsync(cancellationToken)); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
} | ||
|
||
public async Task BatchUpdateAsync(int totalRecords, CancellationToken cancellationToken = default(CancellationToken)) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = await (s.Query<VerySimple>().ToListAsync(cancellationToken)); | ||
Assert.That(items.Count, Is.EqualTo(totalRecords)); | ||
|
||
foreach (var item in items) | ||
{ | ||
item.Weight += 5; | ||
await (s.UpdateAsync(item, cancellationToken)); | ||
} | ||
|
||
Sfi.Statistics.Clear(); | ||
await (tx.CommitAsync(cancellationToken)); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
} | ||
|
||
public async Task BatchDeleteAsync(int totalRecords, CancellationToken cancellationToken = default(CancellationToken)) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = await (s.Query<VerySimple>().ToListAsync(cancellationToken)); | ||
Assert.That(items.Count, Is.EqualTo(totalRecords)); | ||
|
||
foreach (var item in items) | ||
{ | ||
await (s.DeleteAsync(item, cancellationToken)); | ||
} | ||
|
||
Sfi.Statistics.Clear(); | ||
await (tx.CommitAsync(cancellationToken)); | ||
} | ||
Assert.That(Sfi.Statistics.PrepareStatementCount, Is.EqualTo(1)); | ||
} | ||
|
||
private async Task DbShoudBeEmptyAsync(CancellationToken cancellationToken = default(CancellationToken)) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (var tx = s.BeginTransaction()) | ||
{ | ||
var items = await (s.Query<VerySimple>().ToListAsync(cancellationToken)); | ||
Assert.That(items.Count, Is.EqualTo(0)); | ||
|
||
await (tx.CommitAsync(cancellationToken)); | ||
} | ||
} | ||
|
||
private async Task CleanupAsync(CancellationToken cancellationToken = default(CancellationToken)) | ||
{ | ||
using (var s = Sfi.OpenSession()) | ||
using (s.BeginTransaction()) | ||
{ | ||
await (s.CreateQuery("delete from VerySimple").ExecuteUpdateAsync(cancellationToken)); | ||
await (s.Transaction.CommitAsync(cancellationToken)); | ||
} | ||
} | ||
|
||
private int FindAllOccurrences(string source, string substring) | ||
{ | ||
if (source == null) | ||
{ | ||
return 0; | ||
} | ||
int n = 0, count = 0; | ||
while ((n = source.IndexOf(substring, n, StringComparison.InvariantCulture)) != -1) | ||
{ | ||
n += substring.Length; | ||
++count; | ||
} | ||
return count; | ||
} | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.