diff --git a/StandardPlugin/ADAPTParquet.cs b/StandardPlugin/ADAPTParquet.cs index 0029224..2f28c4e 100644 --- a/StandardPlugin/ADAPTParquet.cs +++ b/StandardPlugin/ADAPTParquet.cs @@ -46,7 +46,7 @@ public void Write(string outputFile) using (var fs = File.Create(outputFile)) { - using (var writer = new ParquetFileWriter(outputFile, dataFields.ToArray(), keyValueMetadata: geometadata)) + using (var writer = new ParquetFileWriter(fs, dataFields.ToArray(), keyValueMetadata: geometadata)) { int startIndex = 0; int remainingRowGroups = ColumnData.Geometries.Count / RowGroupSize; @@ -72,7 +72,7 @@ public void Write(string outputFile) } var geometries = ColumnData.Geometries.Skip(startIndex).Take(RowGroupSize); var geometryWriter = rg.NextColumn().LogicalWriter(); - geometryWriter.WriteBatch(ColumnData.Geometries.Skip(startIndex).Take(RowGroupSize).ToArray()); + geometryWriter.WriteBatch(geometries.ToArray()); } startIndex += RowGroupSize; remainingRowGroups--; diff --git a/StandardPlugin/Extensions.cs b/StandardPlugin/Extensions.cs index c8b3c72..3d96f1d 100644 --- a/StandardPlugin/Extensions.cs +++ b/StandardPlugin/Extensions.cs @@ -64,9 +64,14 @@ public static IEnumerable> Batch(this IEnumerable items, in public static bool CanConvert(string srcUnitCode, string targetUnitCode) { - var targetUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(targetUnitCode); - var sourceUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(srcUnitCode); - return sourceUOM.Dimension == targetUOM.Dimension; + if (RepresentationUnitSystem.InternalUnitSystemManager.Instance.UnitOfMeasures[targetUnitCode] != null && + RepresentationUnitSystem.InternalUnitSystemManager.Instance.UnitOfMeasures[srcUnitCode] != null) + { + var targetUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(targetUnitCode); + var sourceUOM = RepresentationUnitSystem.UnitSystemManager.GetUnitOfMeasure(srcUnitCode); + return sourceUOM.Dimension == targetUOM.Dimension; + } + return false; } public static bool CanConvertInto(this UnitOfMeasure srcUOM, string targetUnitCode) diff --git a/StandardPlugin/WorkRecordExporter.cs b/StandardPlugin/WorkRecordExporter.cs index f2fe52d..25f0563 100644 --- a/StandardPlugin/WorkRecordExporter.cs +++ b/StandardPlugin/WorkRecordExporter.cs @@ -59,6 +59,13 @@ public static IEnumerable Export(ApplicationDataModel.ADM.ApplicationDat } } + private IEnumerable GetMappableOperations(LoggedData loggedData) + { + return loggedData.OperationData.Where(od => od.OperationType != OperationTypeEnum.DataCollection && //Data Collection always represents tractor data in production plugins and won't map to ADAPT Standard types + od.OperationType != OperationTypeEnum.Transport && + od.GetSpatialRecords().Any()); + } + private IEnumerable Export(ApplicationDataModel.ADM.ApplicationDataModel model) { List errors = new List(); @@ -69,60 +76,57 @@ private IEnumerable Export(ApplicationDataModel.ADM.ApplicationDataModel foreach (var fieldLoggedData in fieldIdGroupBy) { //Group the operations with spatial data - foreach (var operationData in fieldLoggedData.OperationData) + foreach (var operationData in GetMappableOperations(fieldLoggedData)) { - if (operationData.GetSpatialRecords().Any()) + Implement implement = new Implement(operationData, model.Catalog, _geometryPositition, _deviceDefinition, _commonExporters.TypeMappings, errors); + + //Output grouping + OperationDefinition operationDefinition = new OperationDefinition(implement, operationData, fieldLoggedData); + var matchingOperation = groupedOperations.FirstOrDefault(x => x.IsMatchingOperation(operationDefinition)); + if (matchingOperation != null) + { + //This is the logical equivalent of another operation + matchingOperation.SourceOperations.Add(new ConstituentSpatialOperation(operationData, implement)); + matchingOperation.SourceLoggedDatas.Add(fieldLoggedData); + } + else { - Implement implement = new Implement(operationData, model.Catalog, _geometryPositition, _deviceDefinition, _commonExporters.TypeMappings, errors); + //First one in this group + //Add the columns for parquet + errors.AddRange(operationDefinition.ColumnData.AddOperationData(operationData, model.Catalog, implement, _commonExporters)); - //Output grouping - OperationDefinition operationDefinition = new OperationDefinition(implement, operationData, fieldLoggedData); - var matchingOperation = groupedOperations.FirstOrDefault(x => x.IsMatchingOperation(operationDefinition)); - if (matchingOperation != null) + if (operationDefinition.ColumnData.Columns.Any()) { - //This is the logical equivalent of another operation - matchingOperation.SourceOperations.Add(new ConstituentSpatialOperation(operationData, implement)); - matchingOperation.SourceLoggedDatas.Add(fieldLoggedData); + groupedOperations.Add(operationDefinition); } else { - //First one in this group - //Add the columns for parquet - errors.AddRange(operationDefinition.ColumnData.AddOperationData(operationData, model.Catalog, implement, _commonExporters)); - - if (operationDefinition.ColumnData.Columns.Any()) - { - groupedOperations.Add(operationDefinition); - } - else - { - //This operation did not match any variable to export - continue; - } + //This operation did not match any variable to export + continue; + } - //Add the variables for the output operation - VariableElement timestamp = new VariableElement() - { - Name = "Timestamp", - DefinitionCode = "Timestamp", - Id = new Id() { ReferenceId = string.Concat(operationDefinition.Key(), "-timestamp") }, - GeoParquetColumnName = "Timestamp" - }; - operationDefinition.VariablesByOutputName.Add("Timestamp", timestamp); - foreach (var dataColumn in operationDefinition.ColumnData.Columns) + //Add the variables for the output operation + VariableElement timestamp = new VariableElement() + { + Name = "Timestamp", + DefinitionCode = "Timestamp", + Id = new Id() { ReferenceId = string.Concat(operationDefinition.Key(), "-timestamp") }, + GeoParquetColumnName = "Timestamp" + }; + operationDefinition.VariablesByOutputName.Add("Timestamp", timestamp); + foreach (var dataColumn in operationDefinition.ColumnData.Columns) + { + if (!operationDefinition.VariablesByOutputName.ContainsKey(dataColumn.TargetName)) { - if (!operationDefinition.VariablesByOutputName.ContainsKey(dataColumn.TargetName)) + VariableElement variable = new VariableElement() { - VariableElement variable = new VariableElement() - { - Name = dataColumn.SrcName, - DefinitionCode = dataColumn.TargetCode, - ProductId = dataColumn.ProductId, - Id = _commonExporters.ExportID(dataColumn.SrcWorkingData.Id), - GeoParquetColumnName = dataColumn.TargetName - }; - operationDefinition.VariablesByOutputName.Add(dataColumn.TargetName, variable); - } + Name = dataColumn.SrcName, + DefinitionCode = dataColumn.TargetCode, + ProductId = dataColumn.ProductId, + Id = _commonExporters.ExportID(dataColumn.SrcWorkingData.Id), + GeoParquetColumnName = dataColumn.TargetName + }; + operationDefinition.VariablesByOutputName.Add(dataColumn.TargetName, variable); } } } @@ -178,7 +182,7 @@ private IEnumerable Export(ApplicationDataModel.ADM.ApplicationDataModel } else { - errors.Add(new Error + errors.Add(new Error { Id = sourceSummary.Id.ReferenceId.ToString(CultureInfo.InvariantCulture), Description = "Did not convert summary; could not map to an operation.", @@ -197,7 +201,7 @@ private IEnumerable Export(ApplicationDataModel.ADM.ApplicationDataModel foreach (var loggedData in fieldIdGroupBy.ToList()) { var timeScopes = _commonExporters.ExportTimeScopes(loggedData.TimeScopes, out var seasonIds); - if (timeScopes != null) + if (timeScopes != null) { timeScopeElements.AddRange(timeScopes); }