From 67938888abd67a7a9cf40a1d70aa84420424d15a Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 17:07:39 -0700 Subject: [PATCH 1/2] More FileLike --- .../etl/ImportGeneticsCalculationsStep.java | 11 +-- .../onprc_billing/pipeline/BillingTask.java | 87 ++----------------- 2 files changed, 15 insertions(+), 83 deletions(-) diff --git a/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java b/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java index d0515affa..02b8c249a 100644 --- a/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java +++ b/GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java @@ -1,7 +1,6 @@ package org.labkey.GeneticsCore.etl; import org.apache.commons.lang3.StringUtils; -import org.apache.xmlbeans.XmlException; import org.jetbrains.annotations.NotNull; import org.labkey.GeneticsCore.GeneticsCoreModule; import org.labkey.api.data.Container; @@ -17,6 +16,8 @@ import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.view.UnauthorizedException; import org.labkey.api.writer.ContainerUser; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.util.Collections; @@ -64,19 +65,19 @@ public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobExcepti throw new PipelineJobException("Must provide the filepath to import data using the KinshipDataPath module property"); } - File pipeDir = new File(pipeDirPath); + FileLike pipeDir = FileSystemLike.wrapFile(new File(pipeDirPath)); if (!pipeDir.exists()) { throw new PipelineJobException("Path does not exist: " + pipeDir.getPath()); } - File kinship = new File(pipeDir, "kinship.txt"); + FileLike kinship = pipeDir.resolveChild("kinship.txt"); if (!kinship.exists()) { throw new PipelineJobException("File does not exist: " + kinship.getPath()); } - File inbreeding = new File(pipeDir, "inbreeding.txt"); + FileLike inbreeding = pipeDir.resolveChild("inbreeding.txt"); if (!inbreeding.exists()) { throw new PipelineJobException("File does not exist: " + inbreeding.getPath()); @@ -94,7 +95,7 @@ public List getRequiredSettings() } @Override - public void setSettings(Map settings) throws XmlException + public void setSettings(Map settings) { } diff --git a/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java b/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java index 9038fd279..9221aa62d 100644 --- a/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java +++ b/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java @@ -15,7 +15,6 @@ */ package org.labkey.onprc_billing.pipeline; -import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.lang3.time.DateUtils; import org.jetbrains.annotations.NotNull; import org.labkey.api.collections.CaseInsensitiveHashMap; @@ -28,7 +27,6 @@ import org.labkey.api.data.ResultsImpl; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Selector; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.Table; @@ -49,11 +47,6 @@ import org.labkey.api.util.GUID; import org.labkey.onprc_billing.ONPRC_BillingSchema; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.sql.ResultSet; -import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -105,11 +98,9 @@ public List getProtocolActionNames() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public BillingTask createTask(PipelineJob job) { - BillingTask task = new BillingTask(this, job); - - return task; + return new BillingTask(this, job); } @Override @@ -198,43 +189,6 @@ private BillingPipelineJobSupport getSupport() return (BillingPipelineJobSupport)getJob(); } - private void writeToTsv(String fileName, List> rows, String[] headers, String[] colNames) throws PipelineJobException - { - File csvFile = new File(getSupport().getAnalysisDir(), fileName + ".txt"); - if (csvFile.exists()) - throw new PipelineJobException("There is already a file with path: " + csvFile.getPath()); - - try - { - csvFile.createNewFile(); - - try (CSVWriter csv = new CSVWriter(new FileWriter(csvFile), '\t')) - { - csv.writeNext(headers); - - if (!rows.isEmpty()) - { - for (Map row : rows) - { - String[] toWrite = new String[colNames.length]; - int i = 0; - for (String colName: colNames) - { - toWrite[i] = getString(row.get(colName)); - i++; - } - - csv.writeNext(toWrite); - } - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - private String _invoiceId = null; private String getOrCreateInvoiceRunRecord() throws PipelineJobException @@ -323,7 +277,7 @@ private void writeToInvoicedItems(List> rows, String categor TableInfo invoicedItems = ONPRC_BillingSchema.getInstance().getSchema().getTable(ONPRC_BillingSchema.TABLE_INVOICED_ITEMS); for (Map row : rows) { - CaseInsensitiveHashMap toInsert = new CaseInsensitiveHashMap(); + CaseInsensitiveHashMap toInsert = new CaseInsensitiveHashMap<>(); toInsert.put("container", getJob().getContainer().getId()); toInsert.put("createdby", getJob().getUser().getUserId()); toInsert.put("created", new Date()); @@ -373,24 +327,6 @@ private void writeToInvoicedItems(List> rows, String categor } } - private String getString(Object val) - { - if (val == null) - { - return ""; - } - else if (val instanceof Date) - { - return _dateFormat.format(val); - } - else if (val instanceof Number) - { - return val.toString(); - } - - return val.toString(); - } - private void leaseFeeProcessing(Container ehrContainer) throws PipelineJobException { getJob().getLogger().info("Caching Lease Fees"); @@ -467,19 +403,14 @@ private List> getRowList(Container c, String schemaName, Str ts.setNamedParameters(params); final List> rows = new ArrayList<>(); - ts.forEach(new Selector.ForEachBlock<>() - { - @Override - public void exec(ResultSet object) throws SQLException + ts.forEach(object -> { + Results rs = new ResultsImpl(object, colKeys); + Map ret = new HashMap<>(); + for (FieldKey fk : colKeys.keySet()) { - Results rs = new ResultsImpl(object, colKeys); - Map ret = new HashMap<>(); - for (FieldKey fk : colKeys.keySet()) - { - ret.put(fk.toString(), rs.getObject(fk)); - } - rows.add(ret); + ret.put(fk.toString(), rs.getObject(fk)); } + rows.add(ret); }); return rows; From 7d3bd19ccd5495db1d39d4ede3dba969c99a0bcc Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 29 Oct 2025 13:41:27 -0700 Subject: [PATCH 2/2] Minor cleanup --- .../onprc_billing/pipeline/BillingTask.java | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java b/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java index 9221aa62d..1f1ded27e 100644 --- a/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java +++ b/onprc_billing/src/org/labkey/onprc_billing/pipeline/BillingTask.java @@ -24,7 +24,6 @@ import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; -import org.labkey.api.data.ResultsImpl; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; @@ -47,6 +46,7 @@ import org.labkey.api.util.GUID; import org.labkey.onprc_billing.ONPRC_BillingSchema; +import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -266,7 +266,7 @@ private String getOrCreateInvoiceRunRecord() throws PipelineJobException "quantity", "unitCost", "totalcost", "rateId", "exemptionId", "creditaccountid", "comment", "transactionType", "sourceRecord", "chargeCategory"}; - private void writeToInvoicedItems(List> rows, String category, String[] colNames, String queryName, boolean allowNullProject) throws PipelineJobException + private void writeToInvoicedItems(List> rows, String[] colNames, String queryName, boolean allowNullProject) throws PipelineJobException { assert colNames.length >= invoicedItemsCols.length; @@ -370,7 +370,7 @@ private void leaseFeeProcessing(Container ehrContainer) throws PipelineJobExcept getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Lease Fees", colNames, queryName, false); + writeToInvoicedItems(rows, colNames, queryName, false); getJob().getLogger().info("Finished Caching Lease Fees"); } @@ -403,15 +403,22 @@ private List> getRowList(Container c, String schemaName, Str ts.setNamedParameters(params); final List> rows = new ArrayList<>(); - ts.forEach(object -> { - Results rs = new ResultsImpl(object, colKeys); - Map ret = new HashMap<>(); - for (FieldKey fk : colKeys.keySet()) + try (Results results = ts.getResults()) + { + while (results.next()) { - ret.put(fk.toString(), rs.getObject(fk)); + Map ret = new HashMap<>(); + for (Map.Entry entry: colKeys.entrySet()) + { + ret.put(entry.getKey().toString(), entry.getValue().getValue(results)); + } + rows.add(ret); } - rows.add(ret); - }); + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } return rows; } @@ -462,7 +469,7 @@ private void perDiemProcessing(Container ehrContainer) throws PipelineJobExcepti List> rows = getRowList(ehrContainer, "onprc_billing", queryName, colNames, params); getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Per Diems", colNames, queryName, false); + writeToInvoicedItems(rows, colNames, queryName, false); getJob().getLogger().info("Finished Caching Per Diem Fees"); } @@ -514,7 +521,7 @@ private void slaPerDiemProcessing() throws PipelineJobException List> rows = getRowList(slaContainer, "onprc_billing", queryName, colNames, params); getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Small Lab Animal Per Diems", colNames, queryName, false); + writeToInvoicedItems(rows, colNames, queryName, false); getJob().getLogger().info("Finished Caching Per Diem Fees"); } @@ -560,7 +567,7 @@ private void proceduresProcessing(Container ehrContainer) throws PipelineJobExce List> rows = getRowList(ehrContainer, "onprc_billing", queryName, colNames, params); getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Procedure Fees", colNames, queryName, false); + writeToInvoicedItems(rows, colNames, queryName, false); getJob().getLogger().info("Finished Caching Procedure Fees"); } @@ -606,7 +613,7 @@ private void labworkProcessing(Container ehrContainer) throws PipelineJobExcepti List> rows = getRowList(ehrContainer, "onprc_billing", queryName, colNames, params); getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Labwork Fees", colNames, queryName, false); + writeToInvoicedItems(rows, colNames, queryName, false); getJob().getLogger().info("Finished Caching Labwork Fees"); } @@ -649,7 +656,7 @@ private void miscChargesProcessing(Container ehrContainer) throws PipelineJobExc List> rows = getRowList(ehrContainer, "onprc_billing", MISC_CHARGES_QUERY, colNames, params); getJob().getLogger().info(rows.size() + " rows found"); - writeToInvoicedItems(rows, "Other Charges", colNames, MISC_CHARGES_QUERY, true); + writeToInvoicedItems(rows, colNames, MISC_CHARGES_QUERY, true); getJob().getLogger().info("Finished Caching Other Charges"); }