diff --git a/src/org/labkey/serverapi/reader/TabLoader.java b/src/org/labkey/serverapi/reader/TabLoader.java index 2e9d3b6226..c437eed4f1 100644 --- a/src/org/labkey/serverapi/reader/TabLoader.java +++ b/src/org/labkey/serverapi/reader/TabLoader.java @@ -141,7 +141,7 @@ public DataLoader createLoader(InputStream is, boolean hasColumnHeaders) throws private String _strQuote = null; private String _strQuoteQuote = null; private boolean _parseQuotes = true; - private final boolean _unescapeBackslashes = true; + private final boolean _unescapeBackslashes = false; private Filter> _mapFilter; // Infer whether there are headers diff --git a/src/org/labkey/test/tests/SampleTypeNameExpressionTest.java b/src/org/labkey/test/tests/SampleTypeNameExpressionTest.java index 92a15c23ff..659af3b727 100644 --- a/src/org/labkey/test/tests/SampleTypeNameExpressionTest.java +++ b/src/org/labkey/test/tests/SampleTypeNameExpressionTest.java @@ -73,8 +73,9 @@ public class SampleTypeNameExpressionTest extends BaseWebDriverTest { private static final String PROJECT_NAME = "SampleType_Name_Expression_Test"; + // Issue 53548: Naming Pattern with a default value containing a () or {} cannot be saved. private static final String DEFAULT_SAMPLE_PARENT_VALUE = "SS" + - EscapeUtil.escapeForNameExpression(TestDataGenerator.randomString(3)); + EscapeUtil.escapeForNameExpression(TestDataGenerator.randomString(3, "{}()_")); private static final String PARENT_SAMPLE_TYPE = "PS" + DOMAIN_TRICKY_CHARACTERS; private static final String PARENT_SAMPLE_TYPE_INPUT = escapeForNameExpression(PARENT_SAMPLE_TYPE); @@ -251,14 +252,14 @@ public void testDeriveFromCommentLikeParents() log("Verify import tsv to create derivative would ignore lines starting with #"); String nameExpression = "${MaterialInputs/" + PARENT_SAMPLE_TYPE_INPUT + "}-child"; - String data = "MaterialInputs/" + PARENT_SAMPLE_TYPE + "\n"; // unencoded header + String data = "MaterialInputs/" + PARENT_SAMPLE_TYPE + "\n"; data += PARENT_SAMPLE_01 + "\n"; SampleTypeHelper sampleHelper = new SampleTypeHelper(this); sampleHelper.createSampleType(new SampleTypeDefinition(sampleTypeName) .setNameExpression(nameExpression), data); - data = "MaterialInputs/" + PARENT_SAMPLE_TYPE_INPUT + "\n"; // "/" encoded in header + data = "MaterialInputs/" + PARENT_SAMPLE_TYPE + "\n"; data += PARENT_SAMPLE_01 + "," + PARENT_SAMPLE_02 + "," + PARENT_SAMPLE_03 + "\n"; // tsv lines starting with # should be ignored data += PARENT_SAMPLE_03 + "\n"; diff --git a/src/org/labkey/test/tests/SampleTypeRenameTest.java b/src/org/labkey/test/tests/SampleTypeRenameTest.java index f5f8d40a16..e61a745ea0 100644 --- a/src/org/labkey/test/tests/SampleTypeRenameTest.java +++ b/src/org/labkey/test/tests/SampleTypeRenameTest.java @@ -102,6 +102,8 @@ public void testSampleTypeFieldRename() throws IOException, CommandException testDataGenerator.addCustomRow(Map.of(FIELD_INT, intVal++)); testDataGenerator.insertRows(); + SearchAdminAPIHelper.waitForIndexer(); + goToProjectHome(); SampleTypeHelper sampleHelper = new SampleTypeHelper(this); UpdateSampleTypePage updatePage = sampleHelper.goToEditSampleType(sampleTypeName); diff --git a/src/org/labkey/test/tests/SampleTypeTest.java b/src/org/labkey/test/tests/SampleTypeTest.java index b797fc7184..b25c892025 100644 --- a/src/org/labkey/test/tests/SampleTypeTest.java +++ b/src/org/labkey/test/tests/SampleTypeTest.java @@ -1745,7 +1745,7 @@ private void importSampleTypeFilePathDataError(String sampleName, String filePat { ImportDataPage importDataPage = new ImportDataPage(getDriver()); final String fileFieldName = "FileField"; - String pasteData = TestDataUtils.tsvStringFromRowMapsEscapeBackslash(List.of(Map.of("Name", sampleName, fileFieldName, filePath)), + String pasteData = TestDataUtils.tsvStringFromRowMaps(List.of(Map.of("Name", sampleName, fileFieldName, filePath)), List.of("Name", fileFieldName), true); importDataPage.setText(pasteData); String error = importDataPage.submitExpectingError(); diff --git a/src/org/labkey/test/tests/component/GridPanelTest.java b/src/org/labkey/test/tests/component/GridPanelTest.java index 033b531f20..cd3ee0f9c1 100644 --- a/src/org/labkey/test/tests/component/GridPanelTest.java +++ b/src/org/labkey/test/tests/component/GridPanelTest.java @@ -29,6 +29,7 @@ import org.labkey.test.util.DataRegionTable; import org.labkey.test.util.SampleTypeHelper; import org.labkey.test.util.TestDataGenerator; +import org.labkey.test.util.data.TestDataUtils; import org.labkey.test.util.exp.SampleTypeAPIHelper; import org.openqa.selenium.WebElement; @@ -1863,37 +1864,26 @@ public void testFilterDialogWithViews() throws IOException, CommandException */ private void validateExportedData(File exportedFile, List> expectedValues, List columns) throws IOException { + List> rowMaps = TestDataUtils.rowMapsFromCsv(exportedFile); - try (CSVReader reader = new CSVReader(Readers.getReader(exportedFile), GridBar.ExportType.CSV.getSeparator())) - { - List columnNames = columns.stream().map(FieldInfo::getName).toList(); - List columnLabels = columns.stream().map(FieldInfo::getLabel).toList(); - List allRows = reader.readAll(); + checker().verifyEquals("Number of rows is not as expected.", expectedValues.size(), rowMaps.size()); + List columnNames = columns.stream().map(FieldInfo::getName).toList(); + List columnLabels = columns.stream().map(FieldInfo::getLabel).toList(); - // Use headerRow, the column names, as keys for the map of actual data. - String[] headerRow = allRows.get(0); + for (int rowIndex = 0; rowIndex < expectedValues.size(); rowIndex++) + { + // Get the expected row value. + Map expectedRowData = expectedValues.get(rowIndex); - checker().verifyEquals("Number of rows is not as expected.", expectedValues.size()+1, allRows.size()); + // Create a map of the actual row data returned. + Map actualRowData = rowMaps.get(rowIndex); - for (int rowIndex = 0; rowIndex < expectedValues.size(); rowIndex++) + // Check the values from the identified columns (exported vs. expected). All exported values are stings + // so need to convert the expected values. + for (int colIndex = 0; colIndex < columnNames.size(); colIndex++) { - // Get the expected row value. - Map expectedRowData = expectedValues.get(rowIndex); - - // Create a map of the actual row data returned. - Map actualRowData = new HashMap<>(); - String[] row = allRows.get(rowIndex+1); - for (int i = 0; i < headerRow.length; i++) - actualRowData.put(headerRow[i], row[i]); - - // Check the values from the identified columns (exported vs. expected). All exported values are stings - // so need to convert the expected values. - for (int colIndex = 0; colIndex < columnNames.size(); colIndex++) - { - checker().verifyEquals(String.format("Value for column '%s' on row %d not as expected.", columnNames.get(colIndex), rowIndex + 1), - expectedRowData.get(columnNames.get(colIndex)).toString(), actualRowData.get(columnLabels.get(colIndex))); - } - + checker().verifyEquals(String.format("Value for column '%s' on row %d not as expected.", columnNames.get(colIndex), rowIndex), + expectedRowData.get(columnNames.get(colIndex)).toString(), actualRowData.get(columnLabels.get(colIndex))); } } @@ -1911,22 +1901,14 @@ private void validateExportedData(File exportedFile, List> e private void validateExportedColumnHeader(File exportedFile, List expectedColumns, List expectedMissingColumns) throws IOException { - try (CSVReader reader = new CSVReader(Readers.getReader(exportedFile), GridBar.ExportType.CSV.getSeparator())) - { - - List allRows = reader.readAll(); - - List actualHeader = Arrays.asList(allRows.get(0)); - - checker().verifyTrue(String.format("Expected column headers '%s' not exported.", expectedColumns), - actualHeader.containsAll(expectedColumns)); - - for(String missingColumn : expectedMissingColumns) - { - checker().verifyFalse(String.format("Column header '%s' was exported and it should not have been.", missingColumn), - actualHeader.contains(missingColumn)); - } + List actualHeader = TestDataUtils.columnHeaderFromCsv(exportedFile); + checker().verifyTrue(String.format("Expected column headers '%s' not exported.", expectedColumns), + actualHeader.containsAll(expectedColumns)); + for(String missingColumn : expectedMissingColumns) + { + checker().verifyFalse(String.format("Column header '%s' was exported and it should not have been.", missingColumn), + actualHeader.contains(missingColumn)); } } diff --git a/src/org/labkey/test/util/TestDataGenerator.java b/src/org/labkey/test/util/TestDataGenerator.java index e2042e4234..c5c5074976 100644 --- a/src/org/labkey/test/util/TestDataGenerator.java +++ b/src/org/labkey/test/util/TestDataGenerator.java @@ -79,7 +79,7 @@ public class TestDataGenerator public static final char ALL_CHARS_PLACEHOLDER = '\u2211'; // '∑' - Used to indicate that all characters from the charset should be used public static final String NON_LATIN_STRING = "\u0438\u0418\uC548\u306F"; // "иИ안は" // chose a Character random from this String - public static final String CHARSET_STRING = "ABCDEFG01234abcdefvxyz~!@#$%^&*()-+=_{}[]|:;\"',.<>" + NON_LATIN_STRING + WIDE_PLACEHOLDER; + public static final String CHARSET_STRING = "ABCDEFG01234abcdefvxyz~!@#$%^&*()-+=_{}[]|\\:;\"',.<>" + NON_LATIN_STRING + WIDE_PLACEHOLDER; public static final String ALPHANUMERIC_STRING = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvxyz"; public static final String DOMAIN_SPECIAL_STRING = "+- _.:&()/"; public static final String ILLEGAL_DOMAIN_NAME_CHARSET = "<>[]{};,`\"~!@#$%^*=|?\\"; @@ -166,7 +166,7 @@ else if (fieldDefinition.getType().equals(FieldDefinition.ColumnType.Boolean)) } else if (fieldDefinition.getType().equals(FieldDefinition.ColumnType.String)) { - entityData.put(key, "Entity " + i); + entityData.put(key, "Entity " + i + randomString(5).trim()); } else if (fieldDefinition.getType().equals(FieldDefinition.ColumnType.TextChoice)) { @@ -602,7 +602,7 @@ public static String randomFieldName(@NotNull String part, @Nullable Integer num // use the characters that we know are encoded in fieldKeys plus characters that we know clients are using // Issue 53197: Field name with double byte character can cause client side exception in Firefox when trying to customize grid view. - String chars = ALL_ILLEGAL_QUERY_KEY_CHARACTERS + " %()=+-[]_|*`'\":;<>?!@#^" + NON_LATIN_STRING + String chars = ALL_ILLEGAL_QUERY_KEY_CHARACTERS + " %()=+-[]_|*`'\":;\\<>?!@#^" + NON_LATIN_STRING + WIDE_PLACEHOLDER + REPEAT_PLACEHOLDER + ALL_CHARS_PLACEHOLDER; int currentTries = 0; diff --git a/src/org/labkey/test/util/data/TestDataUtils.java b/src/org/labkey/test/util/data/TestDataUtils.java index 807d78e64a..5b662c60d1 100644 --- a/src/org/labkey/test/util/data/TestDataUtils.java +++ b/src/org/labkey/test/util/data/TestDataUtils.java @@ -64,10 +64,10 @@ public class TestDataUtils () -> new FieldDefinition("2.0"), () -> new FieldDefinition("12.0"), () -> new FieldDefinition("FAM-Lambda..cp.Rxn."), - () -> new FieldDefinition("VIC-Precision...1"), + () -> new FieldDefinition("VIC\\Precision...1"), () -> new FieldDefinition("Product.Type"), () -> new FieldDefinition("Weight.Balance_%", FieldDefinition.ColumnType.Decimal), - () -> new FieldDefinition("Cumulative.Yield.DCW/Glucose.Consumed_g/g", FieldDefinition.ColumnType.Decimal), + () -> new FieldDefinition("Cumulative.Yield\\DCW/Glucose.Consumed_g/g", FieldDefinition.ColumnType.Decimal), () -> new FieldDefinition("Average.Volume.Productivity_g/L/day", FieldDefinition.ColumnType.Decimal), () -> new FieldDefinition("Cmol.Biomass/Cmol.Glucose.Consumed_%", FieldDefinition.ColumnType.Decimal) ); @@ -276,6 +276,18 @@ public static List> rowMapsFromCsv(File csvFile) throws IOEx } } + public static List columnHeaderFromCsv(File csvFile) throws IOException + { + try (DataLoader loader = new TabLoader.CsvFactory().createLoader(csvFile, true)) + { + List columnHeaders = new ArrayList<>(); + for (var column : loader.getColumns()) + columnHeaders.add(column.name); + + return columnHeaders; + } + } + public static List> rowMapsFromCsv(String tsvString) throws IOException { try (InputStream dataStream = IOUtils.toInputStream(tsvString, StandardCharsets.UTF_8)) @@ -295,12 +307,6 @@ public static String tsvStringFromRowMaps(List> rowMaps, List return stringFromRowMaps(rowMaps, columns, includeHeaders, CSVFormat.TDF); } - public static String tsvStringFromRowMapsEscapeBackslash(List> rowMaps, List columns, - boolean includeHeaders) - { - return stringFromRowMaps(rowMaps, columns, includeHeaders, CSVFormat.MYSQL); - } - public static List> mapsFromRows(List> allRows) { List> rowMaps = new ArrayList<>(); @@ -471,12 +477,6 @@ public static File writeRowsToTsv(String fileName, List> rows) throw return writeRowsToFile(fileName, rows, CSVFormat.TDF); } - public static File writeRowsToTsvEscapeBackslash(String fileName, List> rows) throws IOException - { - return writeRowsToFile(fileName, rows, CSVFormat.MYSQL); - } - - public static File writeRowsToCsv(String fileName, List> rows) throws IOException { return writeRowsToFile(fileName, rows, CSVFormat.DEFAULT);