From 6b01d921d92e903f7eb114c8a3b073b46a944fc1 Mon Sep 17 00:00:00 2001 From: John Wiseman Date: Wed, 18 Dec 2019 23:01:25 -0800 Subject: [PATCH] Increase writer batch size. Drops time from 61 secs to 42. --- cleanbasestation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cleanbasestation.py b/cleanbasestation.py index 911969f..0c420dc 100644 --- a/cleanbasestation.py +++ b/cleanbasestation.py @@ -27,7 +27,7 @@ def process_changes(db_path, queue): sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?' db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS'])) num_executes += 1 - if num_executes == 10000: + if num_executes == 100000: db.commit() end_time = time.time() print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time))) @@ -122,7 +122,7 @@ def fix_db(db_in_path, db_out_path): 'RegisteredOwners': new_ac_regd_owners}, True, 10) - if num_records % 10000 == 0: + if num_records % 100000 == 0: print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records)) queue.put('DONE') print('Reader: Finished')