From 39a0c54ba0cd84ed886fe84575706f5bd2dba44f Mon Sep 17 00:00:00 2001 From: Charlie Date: Tue, 16 Dec 2025 21:47:32 -0500 Subject: [PATCH 1/3] Add bacteremia isolate ingestion test --- tests/test_ingest.py | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 4944f71..ad88b0d 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -73,3 +73,114 @@ def test_ingest_accepts_path_strings(): session.close() engine.dispose() + + +def test_ingest_bacteremia_example(tmp_path): + engine = create_engine("sqlite:///:memory:") + Session = sessionmaker(bind=engine) + session = Session() + Base.metadata.create_all(engine) + + bacteremia_tsv = """SampleID\tsample species\tReceived by mARC\tCryobanking\tsample_source\tNote\tTechnician\tspecial_collection\tTube Type\tTube Barcode\tBox-name_position\tSubject ID\tSpecimen ID +marc.bacteremia.1\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164439\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.2\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164431\tmARC Bacteremia Isolates Box 1\t2.0\t2.0 +marc.bacteremia.3\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164460\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.4\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164429\tmARC Bacteremia Isolates Box 1\t3.0\t3.0 +marc.bacteremia.5\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164455\tmARC Bacteremia Isolates Box 1\t4.0\t4.0 +marc.bacteremia.6\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164452\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.7\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164446\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.8\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164454\tmARC Bacteremia Isolates Box 1\t5.0\t6.0 +marc.bacteremia.9\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164472\tmARC Bacteremia Isolates Box 1\t6.0\t7.0 +marc.bacteremia.10\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164444\tmARC Bacteremia Isolates Box 1\t6.0\t8.0 +marc.bacteremia.11\tKlebsiella pneumoniae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164428\tmARC Bacteremia Isolates Box 1\t7.0\t9.0 +marc.bacteremia.12\tEscherichia coli\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164468\tmARC Bacteremia Isolates Box 1\t8.0\t10.0 +marc.bacteremia.13\tEnterobacter cloacae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164462\tmARC Bacteremia Isolates Box 1\t9.0\t11.0 +marc.bacteremia.14\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164450\tmARC Bacteremia Isolates Box 1\t10.0\t12.0 +marc.bacteremia.15\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113164467\tmARC Bacteremia Isolates Box 1\t11.0\t13.0 +marc.bacteremia.16\tPseudomonas aeruginosa\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165151\tmARC Bacteremia Isolates Box 1\t12.0\t14.0 +marc.bacteremia.17\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165140\tmARC Bacteremia Isolates Box 1\t13.0\t15.0 +marc.bacteremia.18\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165159\tmARC Bacteremia Isolates Box 1\t14.0\t16.0 +marc.bacteremia.19\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165166\tmARC Bacteremia Isolates Box 1\t14.0\t17.0 +marc.bacteremia.20\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165167\tmARC Bacteremia Isolates Box 1\t14.0\t18.0 +marc.bacteremia.21\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165144\tmARC Bacteremia Isolates Box 1\t15.0\t19.0 +marc.bacteremia.22\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165158\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.23\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165182\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.24\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165177\tmARC Bacteremia Isolates Box 1\t15.0\t21.0 +marc.bacteremia.25\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165181\tmARC Bacteremia Isolates Box 1\t15.0\t22.0 +marc.bacteremia.26\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165162\tmARC Bacteremia Isolates Box 1\t14.0\t23.0 +marc.bacteremia.27\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165180\tmARC Bacteremia Isolates Box 1\t14.0\t24.0 +marc.bacteremia.28\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165161\tmARC Bacteremia Isolates Box 1\t15.0\t25.0 +marc.bacteremia.29\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165153\tmARC Bacteremia Isolates Box 1\t14.0\t26.0 +marc.bacteremia.30\tUnknown\t2022-03-16 00:00:00\t2022-03-16 00:00:00\tblood culture\t\tT'Nia\tBacteremia\ta\tNA2113165169\tmARC Bacteremia Isolates Box 1\t17.0\t27.0 +marc.bacteremia.1\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164442\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.2\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164449\tmARC Bacteremia Isolates Box 1\t2.0\t2.0 +marc.bacteremia.3\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164427\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.4\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113163791\tmARC Bacteremia Isolates Box 1\t3.0\t3.0 +marc.bacteremia.5\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164465\tmARC Bacteremia Isolates Box 1\t4.0\t4.0 +marc.bacteremia.6\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164435\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.7\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164456\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.8\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164445\tmARC Bacteremia Isolates Box 1\t5.0\t6.0 +marc.bacteremia.9\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164447\tmARC Bacteremia Isolates Box 1\t6.0\t7.0 +marc.bacteremia.10\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164457\tmARC Bacteremia Isolates Box 1\t6.0\t8.0 +marc.bacteremia.11\tKlebsiella pneumoniae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164463\tmARC Bacteremia Isolates Box 1\t7.0\t9.0 +marc.bacteremia.12\tEscherichia coli\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164434\tmARC Bacteremia Isolates Box 1\t8.0\t10.0 +marc.bacteremia.13\tEnterobacter cloacae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164470\tmARC Bacteremia Isolates Box 1\t9.0\t11.0 +marc.bacteremia.14\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164432\tmARC Bacteremia Isolates Box 1\t10.0\t12.0 +marc.bacteremia.15\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113164453\tmARC Bacteremia Isolates Box 1\t11.0\t13.0 +marc.bacteremia.16\tPseudomonas aeruginosa\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165142\tmARC Bacteremia Isolates Box 1\t12.0\t14.0 +marc.bacteremia.17\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165147\tmARC Bacteremia Isolates Box 1\t13.0\t15.0 +marc.bacteremia.18\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165143\tmARC Bacteremia Isolates Box 1\t14.0\t16.0 +marc.bacteremia.19\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165174\tmARC Bacteremia Isolates Box 1\t14.0\t17.0 +marc.bacteremia.20\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165173\tmARC Bacteremia Isolates Box 1\t14.0\t18.0 +marc.bacteremia.21\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165183\tmARC Bacteremia Isolates Box 1\t15.0\t19.0 +marc.bacteremia.22\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165179\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.23\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165165\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.24\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165178\tmARC Bacteremia Isolates Box 1\t15.0\t21.0 +marc.bacteremia.25\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165172\tmARC Bacteremia Isolates Box 1\t15.0\t22.0 +marc.bacteremia.26\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165171\tmARC Bacteremia Isolates Box 1\t14.0\t23.0 +marc.bacteremia.27\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165163\tmARC Bacteremia Isolates Box 1\t14.0\t24.0 +marc.bacteremia.28\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165152\tmARC Bacteremia Isolates Box 1\t15.0\t25.0 +marc.bacteremia.29\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165164\tmARC Bacteremia Isolates Box 1\t14.0\t26.0 +marc.bacteremia.30\tUnknown\t2022-03-16 00:00:00\t2022-03-16 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tb\tNA2113165157\tmARC Bacteremia Isolates Box 1\t17.0\t27.0 +marc.bacteremia.1\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113159912\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.2\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164461\tmARC Bacteremia Isolates Box 1\t2.0\t2.0 +marc.bacteremia.3\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164458\tmARC Bacteremia Isolates Box 1\t1.0\t1.0 +marc.bacteremia.4\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164471\tmARC Bacteremia Isolates Box 1\t3.0\t3.0 +marc.bacteremia.5\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164466\tmARC Bacteremia Isolates Box 1\t4.0\t4.0 +marc.bacteremia.6\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113161266\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.7\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164451\tmARC Bacteremia Isolates Box 1\t2.0\t5.0 +marc.bacteremia.8\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164464\tmARC Bacteremia Isolates Box 1\t5.0\t6.0 +marc.bacteremia.9\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164436\tmARC Bacteremia Isolates Box 1\t6.0\t7.0 +marc.bacteremia.10\tUnknown\t2022-03-04 00:00:00\t2022-03-04 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113163803\tmARC Bacteremia Isolates Box 1\t6.0\t8.0 +marc.bacteremia.11\tKlebsiella pneumoniae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164440\tmARC Bacteremia Isolates Box 1\t7.0\t9.0 +marc.bacteremia.12\tEscherichia coli\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164473\tmARC Bacteremia Isolates Box 1\t8.0\t10.0 +marc.bacteremia.13\tEnterobacter cloacae\t2022-03-07 00:00:00\t2022-03-07 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164430\tmARC Bacteremia Isolates Box 1\t9.0\t11.0 +marc.bacteremia.14\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164433\tmARC Bacteremia Isolates Box 1\t10.0\t12.0 +marc.bacteremia.15\tUnknown\t2022-03-09 00:00:00\t2022-03-09 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113164437\tmARC Bacteremia Isolates Box 1\t11.0\t13.0 +marc.bacteremia.16\tPseudomonas aeruginosa\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165146\tmARC Bacteremia Isolates Box 1\t12.0\t14.0 +marc.bacteremia.17\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165139\tmARC Bacteremia Isolates Box 1\t13.0\t15.0 +marc.bacteremia.18\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165148\tmARC Bacteremia Isolates Box 1\t14.0\t16.0 +marc.bacteremia.19\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165149\tmARC Bacteremia Isolates Box 1\t14.0\t17.0 +marc.bacteremia.20\tStaphylococcus epidermidis\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165170\tmARC Bacteremia Isolates Box 1\t14.0\t18.0 +marc.bacteremia.21\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165176\tmARC Bacteremia Isolates Box 1\t15.0\t19.0 +marc.bacteremia.22\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165138\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.23\tEscherichia coli\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165185\tmARC Bacteremia Isolates Box 1\t16.0\t20.0 +marc.bacteremia.24\tStaphylococcus aureus\t2022-03-11 00:00:00\t2022-03-11 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165156\tmARC Bacteremia Isolates Box 1\t15.0\t21.0 +marc.bacteremia.25\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165154\tmARC Bacteremia Isolates Box 1\t15.0\t22.0 +marc.bacteremia.26\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165184\tmARC Bacteremia Isolates Box 1\t14.0\t23.0 +marc.bacteremia.27\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165160\tmARC Bacteremia Isolates Box 1\t14.0\t24.0 +marc.bacteremia.28\tStaphylococcus aureus\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165175\tmARC Bacteremia Isolates Box 1\t15.0\t25.0 +marc.bacteremia.29\tUnknown\t2022-03-14 00:00:00\t2022-03-14 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165150\tmARC Bacteremia Isolates Box 1\t14.0\t26.0 +marc.bacteremia.30\tUnknown\t2022-03-16 00:00:00\t2022-03-16 00:00:00\tblood culture\t\tT'Nia\tBacteremia\tc\tNA2113165168\tmARC Bacteremia Isolates Box 1\t17.0\t27.0 +""" + + tsv_path = tmp_path / "bacteremia.tsv" + tsv_path.write_text(bacteremia_tsv) + + ingest_from_tsvs(isolates=str(tsv_path), yes=True, session=session) + + assert len(get_isolates(session)) == 30 + assert len(get_aliquots(session)) == 90 + + session.close() + engine.dispose() From 824dc8f1bbd45d946353739f74fc8a3bd3d434f0 Mon Sep 17 00:00:00 2001 From: Charlie Date: Tue, 16 Dec 2025 21:57:22 -0500 Subject: [PATCH 2/3] Handle duplicate isolate rows without false conflicts --- marc_db/ingest.py | 19 ++++++++++++------- tests/test_ingest.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/marc_db/ingest.py b/marc_db/ingest.py index 4c444cd..8717219 100644 --- a/marc_db/ingest.py +++ b/marc_db/ingest.py @@ -73,15 +73,20 @@ def _ingest_isolates(df: pd.DataFrame, session: Session): isolates["cryobanking_date"], errors="coerce" ).dt.date - added = [] + added = {} for _, row in isolates.iterrows(): - i = Isolate(**row.to_dict()) - if i.sample_id in {iso.sample_id for iso in added}: - if i != next(iso for iso in added if iso.sample_id == i.sample_id): - print(f"Conflicting isolate data for SampleID {i.sample_id}") + isolate_kwargs = row.to_dict() + sample_id = isolate_kwargs["sample_id"] + + existing = added.get(sample_id) + if existing: + if existing != isolate_kwargs: + print(f"Conflicting isolate data for SampleID {sample_id}") continue - session.add(i) - added.append(i) + + isolate = Isolate(**isolate_kwargs) + session.add(isolate) + added[sample_id] = isolate_kwargs aliquot_df = df[["Tube Barcode", "Box-name_position", "SampleID"]].copy() aliquot_df.columns = ["tube_barcode", "box_name", "isolate_id"] diff --git a/tests/test_ingest.py b/tests/test_ingest.py index ad88b0d..15d41d1 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -75,6 +75,25 @@ def test_ingest_accepts_path_strings(): engine.dispose() +def test_duplicate_isolate_rows_do_not_warn_when_identical(capsys): + engine = create_engine("sqlite:///:memory:") + Session = sessionmaker(bind=engine) + session = Session() + Base.metadata.create_all(engine) + + isolates_df = pd.read_csv(data_dir / "test_multi_aliquot.tsv", sep="\t") + + ingest_from_tsvs(isolates=isolates_df, yes=True, session=session) + + captured = capsys.readouterr() + assert "Conflicting isolate data" not in captured.out + assert len(get_isolates(session)) == 2 + assert len(get_aliquots(session)) == 5 + + session.close() + engine.dispose() + + def test_ingest_bacteremia_example(tmp_path): engine = create_engine("sqlite:///:memory:") Session = sessionmaker(bind=engine) From 074ea709d3a5c6b6bbd342513d721192aa023603 Mon Sep 17 00:00:00 2001 From: Ulthran Date: Wed, 17 Dec 2025 15:57:39 -0500 Subject: [PATCH 3/3] Coerce on ingest --- marc_db/ingest.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/marc_db/ingest.py b/marc_db/ingest.py index 8717219..219eb24 100644 --- a/marc_db/ingest.py +++ b/marc_db/ingest.py @@ -38,7 +38,9 @@ def _ensure_required_columns(df: pd.DataFrame, required: Iterable[str]): raise ValueError(f"Missing required column(s): {', '.join(missing)}") -def _load_dataframe(data: Optional[Union[pd.DataFrame, Path, str]]) -> Optional[pd.DataFrame]: +def _load_dataframe( + data: Optional[Union[pd.DataFrame, Path, str]] +) -> Optional[pd.DataFrame]: if data is None or isinstance(data, pd.DataFrame): return data return pd.read_csv(Path(data), sep="\t") @@ -66,12 +68,24 @@ def _ingest_isolates(df: pd.DataFrame, session: Session): "received_date", "cryobanking_date", ] + isolates["subject_id"] = pd.to_numeric( + isolates["subject_id"], errors="coerce" + ).astype("Int64") + isolates["specimen_id"] = pd.to_numeric( + isolates["specimen_id"], errors="coerce" + ).astype("Int64") isolates["received_date"] = pd.to_datetime( isolates["received_date"], errors="coerce" ).dt.date + isolates["received_date"] = isolates["received_date"].apply( + lambda x: None if pd.isna(x) else x + ) isolates["cryobanking_date"] = pd.to_datetime( isolates["cryobanking_date"], errors="coerce" ).dt.date + isolates["cryobanking_date"] = isolates["cryobanking_date"].apply( + lambda x: None if pd.isna(x) else x + ) added = {} for _, row in isolates.iterrows(): @@ -85,13 +99,31 @@ def _ingest_isolates(df: pd.DataFrame, session: Session): continue isolate = Isolate(**isolate_kwargs) + if not isinstance(isolate.subject_id, int) or not isinstance( + isolate.specimen_id, int + ): + print( + f"Invalid subject_id or specimen_id for SampleID {sample_id}: {isolate.subject_id}, {isolate.specimen_id}" + ) session.add(isolate) + try: + session.flush() + except Exception as e: + print(f"Error adding isolate with SampleID {sample_id}: {e}") + session.rollback() + continue added[sample_id] = isolate_kwargs aliquot_df = df[["Tube Barcode", "Box-name_position", "SampleID"]].copy() aliquot_df.columns = ["tube_barcode", "box_name", "isolate_id"] for _, row in aliquot_df.iterrows(): session.add(Aliquot(**row.to_dict())) + try: + session.flush() + except Exception as e: + print(f"Error adding aliquot for SampleID {row.isolate_id}: {e}") + session.rollback() + continue def _ingest_assemblies(