Skip to content

Commit 8b0b868

Browse files
gmagnugmagnu
committed
fix(ENGKNOW-2961): Feature flag, and refactor.
1 parent 395c96f commit 8b0b868

File tree

6 files changed

+124
-65
lines changed

6 files changed

+124
-65
lines changed

‎gortools/src/test/java/gorsat/UTestGorWrite.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.gorpipe.gor.driver.GorDriverConfig;
3131
import org.gorpipe.gor.driver.linkfile.LinkFile;
3232
import org.gorpipe.gor.driver.linkfile.LinkFileMeta;
33+
import org.gorpipe.gor.driver.linkfile.LinkFileV1;
3334
import org.gorpipe.gor.driver.meta.DataType;
3435
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
3536
import org.gorpipe.gor.model.BaseMeta;
@@ -80,7 +81,7 @@ public void setupTest() throws IOException {
8081
tempRootPath = tempRoot.getRoot().toPath();
8182

8283
var meta = new LinkFileMeta();
83-
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
84+
meta.loadAndMergeMeta(LinkFileV1.getDefaultMetaContent());
8485
meta.setProperty(BaseMeta.HEADER_SERIAL_KEY, "1");
8586
defaultV1LinkFileHeader = meta.formatHeader();
8687
}
@@ -158,19 +159,21 @@ public void testWritePathWithExistingBadLinkFile() throws IOException {
158159
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
159160
Files.writeString(link, "");
160161
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
161-
var linkUrl = LinkFile.load(new FileSource(link)).getLatestEntryUrl();
162-
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkUrl);
162+
var linkFile = LinkFile.load(new FileSource(link));
163+
Assert.assertEquals("1", linkFile.getMeta().getVersion());
164+
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
163165
}
164166

165167
@Test
166168
public void testWritePathWithExistingBadVersionedLinkFile() throws IOException {
167-
Path p = Paths.get("../tests/data/gor/dbsnp_test.gor");
168-
Files.copy(p, workDirPath.resolve("dbsnp.gor"));
169-
Files.writeString(workDirPath.resolve("dbsnp3.gor.link"), "");
169+
Path link = workDirPath.resolve("dbsnp3.gor.link");
170+
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
171+
Files.writeString(link, "## VERSION = 1");
170172
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
171173

172-
Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).startsWith(
173-
defaultV1LinkFileHeader + workDirPath.resolve("dbsnp2.gor") + "\t"));
174+
var linkFile = LinkFile.load(new FileSource(link));
175+
Assert.assertEquals("1", linkFile.getMeta().getVersion());
176+
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
174177
}
175178

176179
@Test

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,39 @@ public abstract class LinkFile {
6666

6767
public static LinkFile load(StreamSource source) throws IOException {
6868
var content = loadContentFromSource(source);
69-
return create(source, content);
69+
var meta = LinkFileMeta.createOrLoad(content, null, false);
70+
return create(source, meta, content);
7071
}
7172

7273
public static LinkFile create(StreamSource source, String content) {
73-
var meta = LinkFileMeta.createAndLoad(content);
74+
var meta = LinkFileMeta.createOrLoad(content, null, true);
75+
return create(source, meta, content);
76+
}
7477

75-
if ("0".equals(meta.getVersion())) {
76-
return new LinkFileV0(source, meta, content);
77-
} else {
78-
return new LinkFileV1(source, meta, content);
79-
}
78+
public static LinkFile create(StreamSource source, LinkFileMeta meta, String content) {
79+
return switch (meta.getVersion()) {
80+
case "0" -> new LinkFileV0(source, meta, content);
81+
case "1" -> new LinkFileV1(source, meta, content);
82+
default -> throw new GorResourceException("Unsupported link file version: " + meta.getVersion(), source.getFullPath());
83+
};
84+
}
85+
86+
public static LinkFile loadV0(StreamSource source) throws IOException {
87+
var content = loadContentFromSource(source);
88+
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
89+
}
90+
91+
public static LinkFile loadV1(StreamSource source) throws IOException {
92+
var content = loadContentFromSource(source);
93+
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
94+
}
95+
96+
public static LinkFile createV0(StreamSource source, String content) throws IOException {
97+
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
98+
}
99+
100+
public static LinkFile createV1(StreamSource source, String content) throws IOException {
101+
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
80102
}
81103

82104
public static String validateAndUpdateLinkFileName(String linkFilePath) {
@@ -148,15 +170,6 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
148170
protected final LinkFileMeta meta;
149171
protected final List<LinkFileEntry> entries; // Entries sorted by time (oldest first)
150172

151-
/**
152-
* Create a new link file from source and content.
153-
*
154-
* @param source the source to create the link file from
155-
* @param content the content of the link file, can be empty or null to create an empty link file.
156-
*/
157-
protected LinkFile(StreamSource source, String content) {
158-
this(source, LinkFileMeta.createAndLoad(content), content);
159-
}
160173

161174
protected LinkFile(StreamSource source, LinkFileMeta meta, String content) {
162175
this.source = source;
@@ -321,6 +334,7 @@ private void save(OutputStream os, long timestamp) {
321334
.filter(entry -> entry.timestamp() <= 0 || currentTimestamp - entry.timestamp() <= getEntriesAgeMax())
322335
.forEach(entry -> content.append(entry.format()).append("\n"));
323336
}
337+
324338
try {
325339
os.write(content.toString().getBytes());
326340
} catch (IOException e) {

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import org.gorpipe.gor.model.FileReader;
66
import org.gorpipe.util.Strings;
77

8+
import java.util.stream.Collectors;
9+
810
public class LinkFileMeta extends BaseMeta {
911

1012
// Max number of entries to keep track of in the link file.
@@ -16,24 +18,39 @@ public class LinkFileMeta extends BaseMeta {
1618
// Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false).
1719
public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED";
1820

19-
public static final String[] DEFAULT_TABLE_HEADER = new String[] {"File", "Timestamp", "MD5", "Serial", "Info"};
21+
private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1");
2022

2123
public static final int DEFAULT_ENTRIES_COUNT_MAX = 100;
2224
public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE;
2325

24-
public static LinkFileMeta createAndLoad(String metaContent) {
26+
/**
27+
* Create or load link file meta from content.
28+
* @param content
29+
* @param version version if known, otherwise null. Only used if content is null or empty.
30+
* @param isNew true if creating new link file meta, false if loading existing.
31+
* @return
32+
*/
33+
public static LinkFileMeta createOrLoad(String content, String version, boolean isNew) {
34+
var metaContent = !Strings.isNullOrEmpty(content) ? content.lines().filter(line -> line.startsWith("#")).collect(Collectors.joining("\n")) : "";
2535
LinkFileMeta meta = new LinkFileMeta();
26-
if (Strings.isNullOrEmpty(metaContent)) {
27-
meta.loadAndMergeMeta(getDefaultMetaContent());
28-
} else {
29-
meta.loadAndMergeMeta(metaContent);
36+
if (Strings.isNullOrEmpty(metaContent) ) {
37+
// No meta, determine version to use
38+
if (Strings.isNullOrEmpty(version)) {
39+
version = Strings.isNullOrEmpty(content) || isNew ? DEFAULT_VERSION : LinkFileV0.VERSION;
40+
}
41+
42+
metaContent = switch(version) {
43+
case "0" -> LinkFileV0.getDefaultMetaContent();
44+
case "1" -> LinkFileV1.getDefaultMetaContent();
45+
default -> throw new IllegalArgumentException("Unsupported link file version: " + meta.getVersion());
46+
};
3047
}
48+
meta.loadAndMergeMeta(metaContent);
3149
return meta;
3250
}
3351

3452
public LinkFileMeta() {
3553
super();
36-
setFileHeader(DEFAULT_TABLE_HEADER);
3754
saveHeaderLine = true;
3855
}
3956

@@ -63,13 +80,8 @@ public void setEntriesAgeMax(int entriesAgeMax) {
6380

6481
@Override
6582
public String getVersion() {
66-
return getProperty(HEADER_VERSION_KEY, "0");
83+
return getProperty(HEADER_VERSION_KEY, DEFAULT_VERSION);
6784
}
6885

69-
public static String getDefaultMetaContent() {
70-
return String.format("""
71-
## SERIAL = 0
72-
## VERSION = 1
73-
""");
74-
}
86+
7587
}

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,8 @@
1111
*/
1212
public class LinkFileV0 extends LinkFile {
1313

14-
/**
15-
* Load from a source, if it exists, otherwise create an empty link file.
16-
*
17-
* @param source the source to load from
18-
*/
19-
public LinkFileV0(StreamSource source) throws IOException {
20-
super(source, loadContentFromSource(source));
21-
}
14+
public static final String VERSION = "0";
15+
2216

2317
protected LinkFileV0(StreamSource source, LinkFileMeta meta, String content) {
2418
super(source, meta, content);
@@ -40,4 +34,10 @@ public LinkFile appendEntry(String link, String md5, String info, FileReader rea
4034
entries.add(new LinkFileEntryV0(link));
4135
return this;
4236
}
37+
38+
public static String getDefaultMetaContent() {
39+
return String.format("""
40+
## VERSION = 0
41+
""");
42+
}
4343
}

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,12 @@
1212
*/
1313
public class LinkFileV1 extends LinkFile {
1414

15+
public static final String VERSION = "1";
16+
public static final String DEFAULT_TABLE_HEADER = "#File\tTimestamp\tMD5\tSerial\tInfo";
17+
1518
private static boolean allowOverwriteOfTargets
1619
= Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "false"));
1720

18-
/**
19-
* Load from a source, if it exists, otherwise create an empty link file.
20-
*
21-
* @param source the source to load from
22-
*/
23-
public LinkFileV1(StreamSource source) throws IOException {
24-
super(source, loadContentFromSource(source));
25-
checkDefaultMeta();
26-
}
27-
2821
protected LinkFileV1(StreamSource source, LinkFileMeta meta, String content) {
2922
super(source, meta, content);
3023
checkDefaultMeta();
@@ -79,9 +72,16 @@ private boolean canReuseEntryWithSameUrl(LinkFileEntry oldEntry, LinkFileEntry n
7972
}
8073

8174
private void checkDefaultMeta() {
82-
if (!meta.containsProperty(BaseMeta.HEADER_VERSION_KEY)) {
83-
getMeta().loadAndMergeMeta(LinkFileMeta.getDefaultMetaContent());
84-
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
75+
if (!meta.getVersion().equals(VERSION)) {
76+
meta.loadAndMergeMeta(getDefaultMetaContent());
8577
}
8678
}
79+
80+
public static String getDefaultMetaContent() {
81+
return String.format("""
82+
## SERIAL = 0
83+
## VERSION = %s
84+
%s
85+
""", VERSION, DEFAULT_TABLE_HEADER);
86+
}
8787
}

‎model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
import org.gorpipe.gor.driver.providers.stream.sources.StreamSource;
66
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
77
import org.junit.Before;
8+
import org.junit.Ignore;
89
import org.junit.Rule;
910
import org.junit.Test;
1011
import org.junit.contrib.java.lang.system.EnvironmentVariables;
12+
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
1113
import org.junit.rules.TemporaryFolder;
1214

1315
import java.io.ByteArrayInputStream;
@@ -30,6 +32,10 @@ public class LinkFileTest {
3032
public final EnvironmentVariables environmentVariables
3133
= new EnvironmentVariables();
3234

35+
@Rule
36+
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
37+
38+
3339
private StreamSource mockSource;
3440
private final String v1LinkFileContent = """
3541
## SERIAL = 1
@@ -55,10 +61,34 @@ public void setUp() {
5561
public void testCreateLinkFile() {
5662
LinkFile linkFile = LinkFile.create(mockSource, v1LinkFileContent);
5763
assertNotNull(linkFile);
64+
assertTrue(linkFile instanceof LinkFileV1);
65+
assertEquals("1", linkFile.getMeta().getVersion());
5866
assertEquals(2, linkFile.getEntries().size());
5967
assertEquals(100, linkFile.getEntriesCountMax());
6068
}
6169

70+
@Test
71+
public void testCreateLinkFileSimple() {
72+
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
73+
assertNotNull(linkFile);
74+
assertTrue(linkFile instanceof LinkFileV1);
75+
assertEquals("1", linkFile.getMeta().getVersion());
76+
assertEquals(1, linkFile.getEntries().size());
77+
assertEquals(100, linkFile.getEntriesCountMax());
78+
}
79+
80+
@Ignore("Fiddly test depending on system properties, ignore for now. Can run in isolation to verify.")
81+
@Test
82+
public void testCreateLinkFileSimpleWithDefault0() {
83+
System.setProperty("gor.driver.link.default.version", "0");
84+
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
85+
assertNotNull(linkFile);
86+
assertTrue(linkFile instanceof LinkFileV0);
87+
assertEquals("0", linkFile.getMeta().getVersion());
88+
assertEquals(1, linkFile.getEntries().size());
89+
assertEquals(100, linkFile.getEntriesCountMax());
90+
}
91+
6292
@Test
6393
public void testLoadLinkFile() throws IOException {
6494
when(mockSource.exists()).thenReturn(true);
@@ -101,7 +131,7 @@ public void testGetTimedPath() {
101131
@Test
102132
public void testSaveNewV1LinkFile() throws IOException {
103133
var linkPath = workPath.resolve("test.link");
104-
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
134+
LinkFile linkFile = LinkFile.createV1(new FileSource(linkPath.toString()), "");
105135
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
106136
linkFile.save();
107137
String savedContent = Files.readString(linkPath);
@@ -112,7 +142,7 @@ public void testSaveNewV1LinkFile() throws IOException {
112142
@Test
113143
public void testSaveNewV0LinkFile() throws IOException {
114144
var linkPath = workPath.resolve("test.link");
115-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
145+
LinkFile linkFile = LinkFile.createV0(new FileSource(linkPath.toString()), "");
116146
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
117147
linkFile.save();
118148
String savedContent = Files.readString(linkPath);
@@ -135,7 +165,7 @@ public void testSaveLinkFileV1ToV1() throws IOException {
135165
public void testSaveLinkFileV0ToV0() throws IOException {
136166
var linkPath = workPath.resolve("test.link");
137167
Files.writeString(linkPath, "a/b/c.gorz");
138-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
168+
LinkFile linkFile = LinkFile.load(new FileSource(linkPath.toString()));
139169
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
140170
linkFile.save();
141171
String savedContent = Files.readString(linkPath);
@@ -146,26 +176,26 @@ public void testSaveLinkFileV0ToV0() throws IOException {
146176
public void testSaveLinkFileV0ToV1() throws IOException {
147177
var linkPath = workPath.resolve("test.link");
148178
Files.writeString(linkPath, "a/b/c.gorz");
149-
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
179+
LinkFile linkFile = LinkFile.loadV1(new FileSource(linkPath.toString()));
150180
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
151181
linkFile.save();
152182
String savedContent = Files.readString(linkPath);
153183
assertTrue(savedContent.contains("## VERSION = 1"));
184+
assertEquals(2, linkFile.getEntries().size());
154185
assertTrue(savedContent.contains(simpleFile));
155186
}
156187

157188
@Test
158189
public void testSaveLinkFileV1ToV0() throws IOException {
159190
var linkPath = workPath.resolve("test.link");
160191
Files.writeString(linkPath, v1LinkFileContent);
161-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
192+
LinkFile linkFile = LinkFile.loadV0(new FileSource(linkPath.toString()));
162193
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
163194
linkFile.save();
164195
String savedContent = Files.readString(linkPath);
165196
assertEquals(simpleFile, savedContent.trim());
166197
}
167198

168-
169199
@Test(expected = IllegalArgumentException.class)
170200
public void testInferDataFileNameFromLinkFile_NullOrEmptyPath() throws Exception {
171201
LinkFile.inferDataFileNameFromLinkFile(new FileSource(""), null);

0 commit comments

Comments
 (0)