tyrel/advisory-circular
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Increase writer batch size. Drops time from 61 secs to 42.

Browse Source
This commit is contained in:
John Wiseman 2019-12-18 23:01:25 -08:00
parent 5046c14693
commit 6b01d921d9
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cleanbasestation.py
View File

@ -27,7 +27,7 @@ def process_changes(db_path, queue):
sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?' sql = 'UPDATE Aircraft SET Type = ?, RegisteredOwners = ? where ModeS = ?'
db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS'])) db.execute(sql, (rec['Type'], rec['RegisteredOwners'], rec['ModeS']))
num_executes += 1 num_executes += 1
if num_executes == 10000: if num_executes == 100000:
db.commit() db.commit()
end_time = time.time() end_time = time.time()
print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time))) print('Writer: Processing %.1f records/sec' % (num_executes / (end_time - start_time)))
@ -122,7 +122,7 @@ def fix_db(db_in_path, db_out_path):
'RegisteredOwners': new_ac_regd_owners}, 'RegisteredOwners': new_ac_regd_owners},
True, True,
10) 10)
if num_records % 10000 == 0: if num_records % 100000 == 0:
print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records)) print('Reader: Processed %s records, fixed %s' % (num_records, num_fixed_records))
queue.put('DONE') queue.put('DONE')
print('Reader: Finished') print('Reader: Finished')