Skip to content

Release package #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kelly-nelson merged 6 commits into from
Nov 6, 2025
Merged

Release package #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e095213
ParquetFileWriter changed to use FileStream rather than filepath. Had…
smith0505 Aug 1, 2025
ab19058
Filtering Operation Types that will not create usable output layers
kelly-nelson Sep 11, 2025
7116da6
Merge pull request #6 from knelson-farmbeltnorth/develop
strhea Sep 24, 2025
3cf0da4
Merge pull request #5 from smith0505/ParquetFilestream
kelly-nelson Sep 25, 2025
c6047b4
Handle missing units
kelly-nelson Oct 31, 2025
add0c4d
Merge pull request #7 from knelson-farmbeltnorth/develop
kelly-nelson Nov 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions StandardPlugin/ADAPTParquet.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public void Write(string outputFile)

using (var fs = File.Create(outputFile))
{
using (var writer = new ParquetFileWriter(outputFile, dataFields.ToArray(), keyValueMetadata: geometadata))
using (var writer = new ParquetFileWriter(fs, dataFields.ToArray(), keyValueMetadata: geometadata))
{
int startIndex = 0;
int remainingRowGroups = ColumnData.Geometries.Count / RowGroupSize;
Expand All @@ -72,7 +72,7 @@ public void Write(string outputFile)
}
var geometries = ColumnData.Geometries.Skip(startIndex).Take(RowGroupSize);
var geometryWriter = rg.NextColumn().LogicalWriter<byte[]>();
geometryWriter.WriteBatch(ColumnData.Geometries.Skip(startIndex).Take(RowGroupSize).ToArray());
geometryWriter.WriteBatch(geometries.ToArray());
}
startIndex += RowGroupSize;
remainingRowGroups--;
Expand Down
11 changes: 8 additions & 3 deletions StandardPlugin/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,14 @@ public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> items, in

public static bool CanConvert(string srcUnitCode, string targetUnitCode)
{
var targetUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(targetUnitCode);
var sourceUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(srcUnitCode);
return sourceUOM.Dimension == targetUOM.Dimension;
if (RepresentationUnitSystem.InternalUnitSystemManager.Instance.UnitOfMeasures[targetUnitCode] != null &&
RepresentationUnitSystem.InternalUnitSystemManager.Instance.UnitOfMeasures[srcUnitCode] != null)
{
var targetUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(targetUnitCode);
var sourceUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(srcUnitCode);
return sourceUOM.Dimension == targetUOM.Dimension;
}
return false;
}

public static bool CanConvertInto(this UnitOfMeasure srcUOM, string targetUnitCode)
Expand Down
96 changes: 50 additions & 46 deletions StandardPlugin/WorkRecordExporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ public static IEnumerable<IError> Export(ApplicationDataModel.ADM.ApplicationDat
}
}

private IEnumerable<OperationData> GetMappableOperations(LoggedData loggedData)
{
return loggedData.OperationData.Where(od => od.OperationType != OperationTypeEnum.DataCollection && //Data Collection always represents tractor data in production plugins and won't map to ADAPT Standard types
od.OperationType != OperationTypeEnum.Transport &&
od.GetSpatialRecords().Any());
}

private IEnumerable<IError> Export(ApplicationDataModel.ADM.ApplicationDataModel model)
{
List<IError> errors = new List<IError>();
Expand All @@ -69,60 +76,57 @@ private IEnumerable<IError> Export(ApplicationDataModel.ADM.ApplicationDataModel
foreach (var fieldLoggedData in fieldIdGroupBy)
{
//Group the operations with spatial data
foreach (var operationData in fieldLoggedData.OperationData)
foreach (var operationData in GetMappableOperations(fieldLoggedData))
{
if (operationData.GetSpatialRecords().Any())
Implement implement = new Implement(operationData, model.Catalog, _geometryPositition, _deviceDefinition, _commonExporters.TypeMappings, errors);

//Output grouping
OperationDefinition operationDefinition = new OperationDefinition(implement, operationData, fieldLoggedData);
var matchingOperation = groupedOperations.FirstOrDefault(x => x.IsMatchingOperation(operationDefinition));
if (matchingOperation != null)
{
//This is the logical equivalent of another operation
matchingOperation.SourceOperations.Add(new ConstituentSpatialOperation(operationData, implement));
matchingOperation.SourceLoggedDatas.Add(fieldLoggedData);
}
else
{
Implement implement = new Implement(operationData, model.Catalog, _geometryPositition, _deviceDefinition, _commonExporters.TypeMappings, errors);
//First one in this group
//Add the columns for parquet
errors.AddRange(operationDefinition.ColumnData.AddOperationData(operationData, model.Catalog, implement, _commonExporters));

//Output grouping
OperationDefinition operationDefinition = new OperationDefinition(implement, operationData, fieldLoggedData);
var matchingOperation = groupedOperations.FirstOrDefault(x => x.IsMatchingOperation(operationDefinition));
if (matchingOperation != null)
if (operationDefinition.ColumnData.Columns.Any())
{
//This is the logical equivalent of another operation
matchingOperation.SourceOperations.Add(new ConstituentSpatialOperation(operationData, implement));
matchingOperation.SourceLoggedDatas.Add(fieldLoggedData);
groupedOperations.Add(operationDefinition);
}
else
{
//First one in this group
//Add the columns for parquet
errors.AddRange(operationDefinition.ColumnData.AddOperationData(operationData, model.Catalog, implement, _commonExporters));

if (operationDefinition.ColumnData.Columns.Any())
{
groupedOperations.Add(operationDefinition);
}
else
{
//This operation did not match any variable to export
continue;
}
//This operation did not match any variable to export
continue;
}

//Add the variables for the output operation
VariableElement timestamp = new VariableElement()
{
Name = "Timestamp",
DefinitionCode = "Timestamp",
Id = new Id() { ReferenceId = string.Concat(operationDefinition.Key(), "-timestamp") },
GeoParquetColumnName = "Timestamp"
};
operationDefinition.VariablesByOutputName.Add("Timestamp", timestamp);
foreach (var dataColumn in operationDefinition.ColumnData.Columns)
//Add the variables for the output operation
VariableElement timestamp = new VariableElement()
{
Name = "Timestamp",
DefinitionCode = "Timestamp",
Id = new Id() { ReferenceId = string.Concat(operationDefinition.Key(), "-timestamp") },
GeoParquetColumnName = "Timestamp"
};
operationDefinition.VariablesByOutputName.Add("Timestamp", timestamp);
foreach (var dataColumn in operationDefinition.ColumnData.Columns)
{
if (!operationDefinition.VariablesByOutputName.ContainsKey(dataColumn.TargetName))
{
if (!operationDefinition.VariablesByOutputName.ContainsKey(dataColumn.TargetName))
VariableElement variable = new VariableElement()
{
VariableElement variable = new VariableElement()
{
Name = dataColumn.SrcName,
DefinitionCode = dataColumn.TargetCode,
ProductId = dataColumn.ProductId,
Id = _commonExporters.ExportID(dataColumn.SrcWorkingData.Id),
GeoParquetColumnName = dataColumn.TargetName
};
operationDefinition.VariablesByOutputName.Add(dataColumn.TargetName, variable);
}
Name = dataColumn.SrcName,
DefinitionCode = dataColumn.TargetCode,
ProductId = dataColumn.ProductId,
Id = _commonExporters.ExportID(dataColumn.SrcWorkingData.Id),
GeoParquetColumnName = dataColumn.TargetName
};
operationDefinition.VariablesByOutputName.Add(dataColumn.TargetName, variable);
}
}
}
Expand Down Expand Up @@ -178,7 +182,7 @@ private IEnumerable<IError> Export(ApplicationDataModel.ADM.ApplicationDataModel
}
else
{
errors.Add(new Error
errors.Add(new Error
{
Id = sourceSummary.Id.ReferenceId.ToString(CultureInfo.InvariantCulture),
Description = "Did not convert summary; could not map to an operation.",
Expand All @@ -197,7 +201,7 @@ private IEnumerable<IError> Export(ApplicationDataModel.ADM.ApplicationDataModel
foreach (var loggedData in fieldIdGroupBy.ToList())
{
var timeScopes = _commonExporters.ExportTimeScopes(loggedData.TimeScopes, out var seasonIds);
if (timeScopes != null)
if (timeScopes != null)
{
timeScopeElements.AddRange(timeScopes);
}
Expand Down