Skip to content

add getNonOverlappingGraphs() and tests #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lawrencecchen wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add getNonOverlappingGraphs() and tests #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/main/java/org/dataspread/sheetanalyzer/SheetAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,15 @@ public static SheetAnalyzer createSheetAnalyzer(Map<String, String[][]> spreadsh
*
* @return
*/
public abstract Map<String, Pair<Map<Ref, List<RefWithMeta>>,
Map<Ref, List<RefWithMeta>>>> getTACODepGraphs();
public abstract Map<String, Pair<Map<Ref, List<RefWithMeta>>, Map<Ref, List<RefWithMeta>>>> getTACODepGraphs();

/**
* Get a TACO graph without overlapping refs for visualization-friendly
* purposes.
*
* @return
*/
public abstract Map<String, Pair<Map<Ref, List<RefWithMeta>>, Map<Ref, List<RefWithMeta>>>> getNonOverlappingGraphs();

/**
* Get the formula clusters
Expand Down
14 changes: 12 additions & 2 deletions src/main/java/org/dataspread/sheetanalyzer/analyzer/SheetAnalyzerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.dataspread.sheetanalyzer.parser.POIParser;
import org.dataspread.sheetanalyzer.SheetAnalyzer;
import org.dataspread.sheetanalyzer.data.CellContent;
import org.dataspread.sheetanalyzer.data.SheetData;
import org.dataspread.sheetanalyzer.util.Pair;
import org.dataspread.sheetanalyzer.util.Ref;

Expand Down Expand Up @@ -83,8 +84,7 @@ public Set<Ref> getDependents(String sheetName, Ref ref) {
}

@Override
public Map<String, Pair<Map<Ref, List<RefWithMeta>>,
Map<Ref, List<RefWithMeta>>>> getTACODepGraphs() {
public Map<String, Pair<Map<Ref, List<RefWithMeta>>, Map<Ref, List<RefWithMeta>>>> getTACODepGraphs() {
Map<String, Pair<Map<Ref, List<RefWithMeta>>, Map<Ref, List<RefWithMeta>>>> tacoDepGraphs = new HashMap<>();
this.depGraphMap.forEach((sheetName, depGraph) -> {
tacoDepGraphs.put(sheetName,
Expand All @@ -93,6 +93,16 @@ Map<Ref, List<RefWithMeta>>>> getTACODepGraphs() {
return tacoDepGraphs;
}

@Override
public Map<String, Pair<Map<Ref, List<RefWithMeta>>, Map<Ref, List<RefWithMeta>>>> getNonOverlappingGraphs() {
Map<String, SheetData> sheetDataMap = this.parser.getSheetData();
this.depGraphMap.forEach((sheetName, depGraph) -> {
SheetData sheetData = sheetDataMap.get(sheetName);
((DependencyGraphTACO) depGraph).pruneOverlappingRefs(sheetData);
});
return getTACODepGraphs();
}

/**
* Returns a map where each key is a sheet name and
* each value is another map. In the nested map, each
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/org/dataspread/sheetanalyzer/data/SheetData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ public class SheetData {
private final Map<Ref, CellWithMeta> refMetadata = new HashMap<>();
private final Set<Ref> accessAreaCache = new HashSet<>();
private final String sheetName;
private int _maxRow;
private int _maxCol;

public SheetData(String sheetName) {
this.sheetName = sheetName;
Expand Down Expand Up @@ -96,6 +98,22 @@ public String getSheetName() {
return this.sheetName;
}

public int getMaxRow() {
return this._maxRow;
}

public int getMaxCol() {
return this._maxCol;
}

public void setMaxRow(int maxRow) {
this._maxRow = maxRow;
}

public void setMaxCol(int maxCol) {
this._maxCol = maxCol;
}

// Notice that in the following methods we return a copy
// of the underlying metadata so that the caller doesn't
// accidently re-assign any of the values.
Expand Down
124 changes: 116 additions & 8 deletions src/main/java/org/dataspread/sheetanalyzer/dependency/DependencyGraphTACO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.dataspread.sheetanalyzer.data.CellContent;
import org.dataspread.sheetanalyzer.data.SheetData;
import org.dataspread.sheetanalyzer.dependency.util.*;
import org.dataspread.sheetanalyzer.util.Pair;
import org.dataspread.sheetanalyzer.util.Ref;
import org.dataspread.sheetanalyzer.util.RefImpl;

import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
Expand All @@ -21,6 +24,7 @@ public class DependencyGraphTACO implements DependencyGraph {
protected Map<Ref, List<RefWithMeta>> precToDepList = new HashMap<>();
protected Map<Ref, List<RefWithMeta>> depToPrecList = new HashMap<>();
private RTree<Ref, Rectangle> _rectToRef = RTree.create();
private Set<Ref> _visitedRefs = new HashSet<>();

private final CompressInfoComparator compressInfoComparator = new CompressInfoComparator();

Expand Down Expand Up @@ -53,20 +57,57 @@ private void getDependentsInternal(Ref precUpdate,
Set<Ref> depUpdateRefSet = findUpdateDepRef(precRef, depRefWithMeta.getRef(),
depRefWithMeta.getEdgeMeta(), realUpdateRef);
depUpdateRefSet.forEach(depUpdateRef -> {
LinkedList<Ref> overlapRef = getNonOverlapRef(resultSet.get(), depUpdateRef);
overlapRef.forEach(olRef -> {
resultSet.set(resultSet.get().add(olRef, RefUtils.refToRect(olRef)));
result.add(olRef);
if (!isDirectDep) {
updateQueue.add(olRef);
}
});
updateResult(result, isDirectDep, resultSet, updateQueue, depUpdateRef);
});
});
}
}
}

private void updateResult(LinkedHashSet<Ref> result, boolean isDirectDep,
AtomicReference<RTree<Ref, Rectangle>> resultSet, Queue<Ref> updateQueue, Ref depUpdateRef) {
LinkedList<Ref> overlapRef = getNonOverlapRef(resultSet.get(), depUpdateRef);
overlapRef.forEach(olRef -> {
resultSet.set(resultSet.get().add(olRef, RefUtils.refToRect(olRef)));
result.add(olRef);
if (!isDirectDep)
updateQueue.add(olRef);
});
}

public Set<Ref> getPrecedents(Ref dependent) {
final boolean isDirectDep = false;
LinkedHashSet<Ref> result = new LinkedHashSet<>();

if (RefUtils.isValidRef(dependent))
getPrecedentInternal(dependent, result, isDirectDep);
return result;
}

private void getPrecedentInternal(Ref depUpdate,
LinkedHashSet<Ref> result,
boolean isDirectPrec) {
AtomicReference<RTree<Ref, Rectangle>> resultSet = new AtomicReference<>(RTree.create());
Queue<Ref> updateQueue = new LinkedList<>();
updateQueue.add(depUpdate);
while (!updateQueue.isEmpty()) {
Ref updateRef = updateQueue.remove();
Iterator<Ref> refIter = findOverlappingRefs(updateRef);
while (refIter.hasNext()) {
Ref depRef = refIter.next();
Ref realUpdateRef = updateRef.getOverlap(depRef);
for (RefWithMeta precRefWithMeta : findPrecs(depRef)) {
Ref precUpdateRef = findUpdatePrecRef(depRef, precRefWithMeta.getRef(),
precRefWithMeta.getEdgeMeta(), realUpdateRef, isDirectPrec);
if (precUpdateRef != null) {

updateResult(result, isDirectPrec, resultSet, updateQueue, precUpdateRef);
}
}
}
}
}

private LinkedList<Ref> getNonOverlapRef(RTree<Ref, Rectangle> resultSet, Ref input) {
LinkedList<Ref> retRefList = new LinkedList<>();
retRefList.addLast(input);
Expand All @@ -81,6 +122,73 @@ private LinkedList<Ref> getNonOverlapRef(RTree<Ref, Rectangle> resultSet, Ref in
return retRefList;
}

public void pruneOverlappingRefs(SheetData sheetData) {
for (int i = 0; i <= sheetData.getMaxCol(); i++) {
for (int j = 0; j < sheetData.getMaxRow(); j++) {
Ref ref = new RefImpl(j, i);
CellContent cellContent = sheetData.getCellContent(ref);
if (_visitedRefs.contains(ref) || !cellContent.isFormula()) {
continue;
}
findMaxOverlapRange(i, j, Direction.TODOWN, sheetData);
findMaxOverlapRange(i, j, Direction.TORIGHT, sheetData);
}
}
}

private void findMaxOverlapRange(int col, int row, Direction direction, SheetData sheetData) {
Set<Ref> overlappingTargetRefs = new HashSet<>();
Ref targetRef = new RefImpl(row, col);
findOverlappingRefs(targetRef).forEachRemaining(overlappingTargetRefs::add);
if (direction != Direction.TODOWN && direction != Direction.TORIGHT) {
throw new IllegalArgumentException("Direction must be either TODOWN or TORIGHT");
}
boolean isDown = direction == Direction.TODOWN;
int start = (isDown ? row : col) + 1;
int max = isDown ? sheetData.getMaxRow() : sheetData.getMaxCol();
for (int i = start; i < max; i++) {
Ref currentRef = new RefImpl(isDown ? i : row, isDown ? col : i);
CellContent cellContent = sheetData.getCellContent(currentRef);
if (!cellContent.isFormula()) {
break;
}
Set<Ref> overlappingCurrentRefs = new HashSet<>();
findOverlappingRefs(currentRef).forEachRemaining(overlappingCurrentRefs::add);
if (!overlappingTargetRefs.equals(overlappingCurrentRefs)) {
// overlappingTargetRefs.retainAll(overlappingCurrentRefs);
overlappingCurrentRefs.retainAll(overlappingTargetRefs);
for (Ref overlappingTargetRef : overlappingCurrentRefs) {
// for (Ref overlappingTargetRef : overlappingTargetRefs) {
Set<RefWithMeta> precedents = new HashSet<>();
findPrecs(overlappingTargetRef).forEach(precedents::add);
for (RefWithMeta precRangeWithMeta : precedents) {
Ref precRef = precRangeWithMeta.getRef();
EdgeMeta edgeMeta = precRangeWithMeta.getEdgeMeta();
List<Pair<Ref, RefWithMeta>> newEdges = deleteOneCell(precRef,
overlappingTargetRef,
edgeMeta,
targetRef);
deleteMemEntry(precRef, overlappingTargetRef, edgeMeta);
Ref deletedRef = new RefImpl(precRef.getRow(), precRef.getColumn());
add(deletedRef, targetRef);
newEdges.forEach(pair -> {
Ref newPrec = pair.first;
Ref newDep = pair.second.getRef();
EdgeMeta newEdgeMeta = pair.second.getEdgeMeta();
if (newDep.getType() == Ref.RefType.CELL) {
add(newPrec, newDep);
} else {
insertMemEntry(newPrec, newDep, newEdgeMeta);
}
});
}
}
} else {
_visitedRefs.add(currentRef);
}
}
}

public long getNumEdges() {
AtomicLong numEdges = new AtomicLong(0);
depToPrecList.forEach((dep, precSet) -> {
Expand Down
10 changes: 4 additions & 6 deletions src/main/java/org/dataspread/sheetanalyzer/parser/POIParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,6 @@ private void parseSpreadsheet() throws SheetNotSupportedException {

private SheetData parseOneSheet(Sheet sheet) throws SheetNotSupportedException {
SheetData sheetData = new SheetData(sheet.getSheetName());
int maxRows = 0;
int maxCols = 0;
for (Row row : sheet) {
for (Cell cell : row) {
if (cell != null) {
Expand All @@ -130,12 +128,12 @@ private SheetData parseOneSheet(Sheet sheet) throws SheetNotSupportedException {
sheetData.addContent(dep, cellContent);
}
}
if (cell.getColumnIndex() > maxCols) {
maxCols = cell.getColumnIndex();
if (cell.getColumnIndex() > sheetData.getMaxCol()) {
sheetData.setMaxCol(cell.getColumnIndex());
}
}
if (row.getRowNum() > maxRows) {
maxRows = row.getRowNum();
if (row.getRowNum() > sheetData.getMaxRow()) {
sheetData.setMaxRow(row.getRowNum());
}
}
return sheetData;
Expand Down
Loading