Skip to content

Commit d48198f

Browse files
antaencantaenc
authored
16.ant.1 (#17)
* #16 - Updates * #16 - Updates * Update CHANGELOG.md #16
1 parent df58376 commit d48198f

33 files changed

+1418
-834
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
[markdownlint](https://dlaa.me/markdownlint/),
77
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88

9+
## [1.1.0] - 2023-12-21
10+
11+
### Changed in 1.1.0
12+
13+
- Updates
14+
915
## [1.0.2] - 2023-03-08
1016

1117
### Removed in 1.0.2

‎Python/Tasks/Deleting/DeleteFutures.py‎

Lines changed: 56 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -6,88 +6,106 @@
66
import os
77
import sys
88
import time
9-
from senzing import G2BadInputException, G2Engine, G2Exception, G2RetryableException, G2UnrecoverableException
9+
from senzing import (
10+
G2BadInputException,
11+
G2Engine,
12+
G2Exception,
13+
G2RetryableException,
14+
G2UnrecoverableException,
15+
)
1016

11-
engine_config_json = os.getenv('SENZING_ENGINE_CONFIGURATION_JSON', None)
17+
engine_config_json = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", None)
1218

1319

1420
def mock_logger(level, exception, error_rec=None):
15-
print(f'\n{level}: {exception}', file=sys.stderr)
21+
print(f"\n{level}: {exception}", file=sys.stderr)
1622
if error_rec:
17-
print(f'{error_rec}', file=sys.stderr)
23+
print(f"{error_rec}", file=sys.stderr)
1824

1925

2026
def del_record(engine, rec_to_del):
2127
record_dict = json.loads(rec_to_del)
22-
data_source = record_dict.get('DATA_SOURCE', None)
23-
record_id = record_dict.get('RECORD_ID', None)
28+
data_source = record_dict.get("DATA_SOURCE", None)
29+
record_id = record_dict.get("RECORD_ID", None)
2430
engine.deleteRecord(data_source, record_id)
2531

2632

2733
def engine_stats(engine):
2834
response = bytearray()
2935
try:
3036
engine.stats(response)
31-
print(f'\n{response.decode()}\n')
32-
except G2RetryableException as ex:
33-
mock_logger('WARN', ex)
34-
except (G2UnrecoverableException, G2Exception) as ex:
35-
mock_logger('CRITICAL', ex)
37+
print(f"\n{response.decode()}\n")
38+
except G2RetryableException as err:
39+
mock_logger("WARN", err)
40+
except G2Exception as err:
41+
mock_logger("CRITICAL", err)
3642
raise
3743

3844

39-
def record_stats(success_recs, prev_time):
40-
print(f'Processed {success_recs} deletes, {int(1000 / (time.time() - prev_time))} records per second')
45+
def record_stats(success, error, prev_time):
46+
print(
47+
f"Processed {success:,} deletes,"
48+
f" {int(1000 / (time.time() - prev_time)):,} records per second,"
49+
f" {error} errors"
50+
)
4151
return time.time()
4252

4353

4454
def futures_del(engine, input_file):
4555
prev_time = time.time()
4656
success_recs = error_recs = 0
4757

48-
with open(input_file, 'r') as in_file:
58+
with open(input_file, "r") as in_file:
4959
with concurrent.futures.ThreadPoolExecutor() as executor:
50-
futures = {executor.submit(del_record, engine, record): record for record in itertools.islice(in_file, executor._max_workers)}
60+
futures = {
61+
executor.submit(del_record, engine, record): record
62+
for record in itertools.islice(in_file, executor._max_workers)
63+
}
5164

5265
while futures:
53-
for f in concurrent.futures.as_completed(futures.keys()):
66+
done, _ = concurrent.futures.wait(
67+
futures, return_when=concurrent.futures.FIRST_COMPLETED
68+
)
69+
for f in done:
5470
try:
5571
f.result()
56-
except G2BadInputException as ex:
57-
mock_logger('ERROR', ex, futures[f])
72+
except (G2BadInputException, json.JSONDecodeError) as err:
73+
mock_logger("ERROR", err, futures[f])
5874
error_recs += 1
59-
except G2RetryableException as ex:
60-
mock_logger('WARN', ex, futures[f])
75+
except G2RetryableException as err:
76+
mock_logger("WARN", err, futures[f])
6177
error_recs += 1
62-
except (G2UnrecoverableException, G2Exception) as ex:
63-
mock_logger('CRITICAL', ex, futures[f])
78+
except (G2UnrecoverableException, G2Exception) as err:
79+
mock_logger("CRITICAL", err, futures[f])
6480
raise
65-
except json.JSONDecodeError as ex:
66-
mock_logger('ERROR', ex, futures[f])
67-
error_recs += 1
6881
else:
69-
success_recs += 1
82+
record = in_file.readline()
83+
if record:
84+
futures[executor.submit(del_record, engine, record)] = (
85+
record
86+
)
7087

88+
success_recs += 1
7189
if success_recs % 1000 == 0:
72-
prev_time = record_stats(success_recs, prev_time)
90+
prev_time = record_stats(
91+
success_recs, error_recs, prev_time
92+
)
7393

7494
if success_recs % 10000 == 0:
7595
engine_stats(engine)
7696
finally:
77-
futures.pop(f)
78-
79-
record = in_file.readline()
80-
if record:
81-
futures[executor.submit(del_record, engine, record)] = record
97+
del futures[f]
8298

83-
print(f'Successfully deleted {success_recs} records, with {error_recs} errors')
99+
print(
100+
f"Successfully deleted {success_recs:,} records, with"
101+
f" {error_recs:,} errors"
102+
)
84103

85104

86105
try:
87106
g2_engine = G2Engine()
88-
g2_engine.init('G2Engine', engine_config_json, False)
89-
futures_del(g2_engine, '../../../Resources/Data/del-10K.json')
107+
g2_engine.init("G2Engine", engine_config_json, False)
108+
futures_del(g2_engine, "../../../Resources/Data/del-10K.json")
90109
g2_engine.destroy()
91-
except (G2BadInputException, G2RetryableException, G2UnrecoverableException, G2Exception) as ex:
92-
print(ex)
93-
sys.exit(-1)
110+
except G2Exception as err:
111+
mock_logger("CRITICAL", err)

‎Python/Tasks/Deleting/DeleteLoop.py‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,51 +3,56 @@
33
import json
44
import os
55
import sys
6-
from senzing import G2BadInputException, G2Engine, G2Exception, G2RetryableException, G2UnrecoverableException
6+
from senzing import (
7+
G2BadInputException,
8+
G2Engine,
9+
G2Exception,
10+
G2RetryableException,
11+
G2UnrecoverableException,
12+
)
713

8-
engine_config_json = os.getenv('SENZING_ENGINE_CONFIGURATION_JSON', None)
14+
engine_config_json = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", None)
915

1016

1117
def mock_logger(level, exception, error_rec=None):
12-
print(f'\n{level}: {exception}', file=sys.stderr)
18+
print(f"\n{level}: {exception}", file=sys.stderr)
1319
if error_rec:
14-
print(f'{error_rec}', file=sys.stderr)
20+
print(f"{error_rec}", file=sys.stderr)
1521

1622

1723
def del_records_from_file(engine, input_file):
18-
success_recs = 0
24+
success_recs = error_recs = 0
1925

20-
with open(input_file, 'r') as file:
26+
with open(input_file, "r") as file:
2127

2228
for rec_to_add in file:
2329
try:
2430
record_dict = json.loads(rec_to_add)
25-
data_source = record_dict.get('DATA_SOURCE', None)
26-
record_id = record_dict.get('RECORD_ID', None)
31+
data_source = record_dict.get("DATA_SOURCE", None)
32+
record_id = record_dict.get("RECORD_ID", None)
2733
engine.deleteRecord(data_source, record_id, rec_to_add)
28-
except G2BadInputException as ex:
29-
mock_logger('ERROR', ex, rec_to_add)
30-
except G2RetryableException as ex:
31-
mock_logger('WARN', ex, rec_to_add)
32-
except (G2UnrecoverableException, G2Exception) as ex:
33-
mock_logger('CRITICAL', ex, rec_to_add)
34+
except (G2BadInputException, json.JSONDecodeError) as err:
35+
mock_logger("ERROR", err, rec_to_add)
36+
error_recs += 1
37+
except G2RetryableException as err:
38+
mock_logger("WARN", err, rec_to_add)
39+
error_recs += 1
40+
except (G2UnrecoverableException, G2Exception) as err:
41+
mock_logger("CRITICAL", err, rec_to_add)
3442
raise
35-
except json.JSONDecodeError as ex:
36-
mock_logger('ERROR', ex, rec_to_add)
3743
else:
3844
success_recs += 1
3945

4046
if success_recs % 1000 == 0:
41-
print(f'Processed {success_recs} deletes')
47+
print(f"Processed {success_recs:,} deletes, with {error_recs:,} errors")
4248

43-
print(f'Successfully deleted {success_recs} records')
49+
print(f"Successfully deleted {success_recs:,} records, with {error_recs:,} errors")
4450

4551

4652
try:
4753
g2_engine = G2Engine()
48-
g2_engine.init('G2Engine', engine_config_json, False)
49-
del_records_from_file(g2_engine, '../../../Resources/Data/del-10K.json')
54+
g2_engine.init("G2Engine", engine_config_json, False)
55+
del_records_from_file(g2_engine, "../../../Resources/Data/del-10K.json")
5056
g2_engine.destroy()
51-
except (G2BadInputException, G2RetryableException, G2UnrecoverableException, G2Exception) as ex:
52-
print(ex)
53-
sys.exit(-1)
57+
except G2Exception as err:
58+
mock_logger("CRITICAL", err)

‎Python/Tasks/Deleting/DeleteWithInfoFutures.py‎

Lines changed: 63 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -6,96 +6,116 @@
66
import os
77
import sys
88
import time
9-
from senzing import G2BadInputException, G2Engine, G2Exception, G2RetryableException, G2UnrecoverableException
9+
from senzing import (
10+
G2BadInputException,
11+
G2Engine,
12+
G2Exception,
13+
G2RetryableException,
14+
G2UnrecoverableException,
15+
)
1016

11-
engine_config_json = os.getenv('SENZING_ENGINE_CONFIGURATION_JSON', None)
17+
engine_config_json = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", None)
1218

1319

1420
def mock_logger(level, exception, error_rec=None):
15-
print(f'\n{level}: {exception}', file=sys.stderr)
21+
print(f"\n{level}: {exception}", file=sys.stderr)
1622
if error_rec:
17-
print(f'{error_rec}', file=sys.stderr)
23+
print(f"{error_rec}", file=sys.stderr)
1824

1925

2026
def del_record(engine, rec_to_del):
2127
with_info = bytearray()
2228
record_dict = json.loads(rec_to_del)
23-
data_source = record_dict.get('DATA_SOURCE', None)
24-
record_id = record_dict.get('RECORD_ID', None)
29+
data_source = record_dict.get("DATA_SOURCE", None)
30+
record_id = record_dict.get("RECORD_ID", None)
2531
engine.deleteRecordWithInfo(data_source, record_id, with_info)
26-
return with_info.decode() + '\n'
32+
return with_info.decode()
2733

2834

2935
def engine_stats(engine):
3036
response = bytearray()
3137
try:
3238
engine.stats(response)
33-
print(f'\n{response.decode()}\n')
34-
except G2RetryableException as ex:
35-
mock_logger('WARN', ex)
36-
except (G2UnrecoverableException, G2Exception) as ex:
37-
mock_logger('CRITICAL', ex)
39+
print(f"\n{response.decode()}\n")
40+
except G2RetryableException as err:
41+
mock_logger("WARN", err)
42+
except G2Exception as err:
43+
mock_logger("CRITICAL", err)
3844
raise
3945

4046

41-
def record_stats(success_recs, prev_time):
42-
print(f'Processed {success_recs} deletes, {int(1000 / (time.time() - prev_time))} records per second')
47+
def record_stats(success, error, prev_time):
48+
print(
49+
f"Processed {success:,} deletes,"
50+
f" {int(1000 / (time.time() - prev_time)):,} records per second,"
51+
f" {error} errors"
52+
)
4353
return time.time()
4454

4555

4656
def futures_del(engine, input_file, output_file):
4757
prev_time = time.time()
4858
success_recs = error_recs = 0
4959

50-
with open(output_file, 'w') as out_file:
51-
with open(input_file, 'r') as in_file:
60+
with open(output_file, "w") as out_file:
61+
with open(input_file, "r") as in_file:
5262
with concurrent.futures.ThreadPoolExecutor() as executor:
53-
futures = {executor.submit(del_record, engine, record): record for record in itertools.islice(in_file, executor._max_workers)}
63+
futures = {
64+
executor.submit(del_record, engine, record): record
65+
for record in itertools.islice(in_file, executor._max_workers)
66+
}
5467

5568
while futures:
56-
for f in concurrent.futures.as_completed(futures.keys()):
69+
done, _ = concurrent.futures.wait(
70+
futures, return_when=concurrent.futures.FIRST_COMPLETED
71+
)
72+
for f in done:
5773
try:
5874
result = f.result()
59-
except G2BadInputException as ex:
60-
mock_logger('ERROR', ex, futures[f])
75+
except (G2BadInputException, json.JSONDecodeError) as err:
76+
mock_logger("ERROR", err, futures[f])
6177
error_recs += 1
62-
except G2RetryableException as ex:
63-
mock_logger('WARN', ex, futures[f])
78+
except G2RetryableException as err:
79+
mock_logger("WARN", err, futures[f])
6480
error_recs += 1
65-
except (G2UnrecoverableException, G2Exception) as ex:
66-
mock_logger('CRITICAL', ex, futures[f])
81+
except (G2UnrecoverableException, G2Exception) as err:
82+
mock_logger("CRITICAL", err, futures[f])
6783
raise
68-
except json.JSONDecodeError as ex:
69-
mock_logger('ERROR', ex, futures[f])
70-
error_recs += 1
7184
else:
72-
success_recs += 1
73-
out_file.write(result)
85+
record = in_file.readline()
86+
if record:
87+
futures[executor.submit(del_record, engine, record)] = (
88+
record
89+
)
90+
91+
out_file.write(f"{result}\n")
7492

93+
success_recs += 1
7594
if success_recs % 1000 == 0:
76-
prev_time = record_stats(success_recs, prev_time)
95+
prev_time = record_stats(
96+
success_recs, error_recs, prev_time
97+
)
7798

7899
if success_recs % 10000 == 0:
79100
engine_stats(engine)
80101
finally:
81-
futures.pop(f)
82-
83-
record = in_file.readline()
84-
if record:
85-
futures[executor.submit(del_record, engine, record)] = record
102+
del futures[f]
86103

87-
print(f'Successfully deleted {success_recs} records, with {error_recs} errors')
88-
print(f'With info responses written to {output_file}')
104+
print(
105+
f"Successfully deleted {success_recs:,} records, with"
106+
f" {error_recs:,} errors"
107+
)
108+
print(f"With info responses written to {output_file}")
89109

90110

91111
try:
92112
g2_engine = G2Engine()
93-
g2_engine.init('G2Engine', engine_config_json, False)
113+
g2_engine.init("G2Engine", engine_config_json, False)
94114
futures_del(
95115
g2_engine,
96-
'../../../Resources/Data/del-10K.json',
97-
'../../../Resources/Output/Del_File_WithInfo.json')
116+
"../../../Resources/Data/del-10K.json",
117+
"../../../Resources/Output/Del_File_WithInfo.json",
118+
)
98119
g2_engine.destroy()
99-
except (G2BadInputException, G2RetryableException, G2UnrecoverableException, G2Exception) as ex:
100-
print(ex)
101-
sys.exit(-1)
120+
except G2Exception as err:
121+
mock_logger("CRITICAL", err)

0 commit comments

Comments
 (0)