Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BOM import #273

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

BOM import #273

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
6a04fd4
Merge pull request #7 from replaysMike/master
shaun-leach May 5, 2024
8bd862c
1st pass KiCAD BOM imporoter
shaun-leach May 5, 2024
90d1f60
1st pass real BOM import
shaun-leach Jun 20, 2024
76f24ec
Add controller and service support for importing a project
shaun-leach Jun 21, 2024
93662fe
Get file and meta info into the controller
shaun-leach Jun 23, 2024
7212b45
Actual import support
shaun-leach Jun 24, 2024
0deb5f3
Remove unneeded using and whitespace
shaun-leach Jun 24, 2024
515296e
Refresh projects page after importing a new BOM
shaun-leach Jun 24, 2024
70d8a9c
CSV import support
shaun-leach Jun 25, 2024
4a1f87a
Properly clean up a failed import
shaun-leach Jun 27, 2024
8ecd0b2
Add translations. Clean up JS code
shaun-leach Jun 29, 2024
091f07a
Merge branch 'master' into kicad-bom-import
replaysMike Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Binner/Binner.Web/ClientApp/src/pages/Boms.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ export function Boms (props) {
>
<span style={{ fontSize: "0.6em" }}>{t('page.exportData.uploadNote', "Drag a document to upload, or click to select files")}</span>
<input {...getInputProps()} />
<div style={{ fontSize: "0.6em" }}>{t('page.exportData.acceptedFileTypes', "Accepted file types: \"*.xls, *.xlsx, *.csv\"")}</div>
<div style={{ fontSize: "0.6em" }}>{t('page.projects.acceptedFileTypes', "Accepted file types: \"*.xls, *.xlsx, *.csv\"")}</div>
</div>
{error && (
<div className="error small">
Expand Down
223 changes: 49 additions & 174 deletions Binner/Library/Binner.Common/IO/CsvBOMImporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,6 @@ public CsvBOMImporter(IStorageProvider storageProvider)
public async Task<ImportResult> ImportAsync(Project project, Stream stream, IUserContext? userContext)
{
var result = new ImportResult();
/* foreach (var table in SupportedTables)
result.RowsImportedByTable.Add(table, 0);
// get the global part types, and the user's custom part types
var partTypes = (await _storageProvider.GetPartTypesAsync(userContext)).ToList();
try
{
stream.Position = 0;
Expand All @@ -43,14 +39,6 @@ public async Task<ImportResult> ImportAsync(Project project, Stream stream, IUse
// remove line breaks
data = data.Replace("\r", "");

var tableName = Path.GetFileNameWithoutExtension(filename);
if (!SupportedTables.Contains(tableName, StringComparer.InvariantCultureIgnoreCase))
{
result.Errors.Add($"Filename '{tableName}' not a supported table name! Filename must be one of the following: {string.Join(",", SupportedTables)} and may optionally contain the schema prefix 'dbo'.");
result.Success = false;
return result;
}

var rows = SplitBoundaries(data, new char[] { '\n' });
if (!rows.Any())
{
Expand All @@ -62,7 +50,7 @@ public async Task<ImportResult> ImportAsync(Project project, Stream stream, IUse
// read csv header
Header? header = null;
var headerRow = rows.First();
if (headerRow.StartsWith("#"))
if (headerRow.StartsWith("\"#\""))
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should likely be if (headerRow.StartsWith("#") || headerRow.StartsWith("\"#\"")) as many CSV formats don't require quoted headers

{
header = new Header(headerRow);
}
Expand All @@ -87,179 +75,66 @@ public async Task<ImportResult> ImportAsync(Project project, Stream stream, IUse
result.Warnings.Add($"[Row {rowNumber}] Row does not contain the same number of columns as the header, skipping...");
continue;
}
switch (tableName.ToLower())
{
case "projects":
{
// import project info
var isProjectIdValid = TryGet<long>(rowData, header, "ProjectId", out var projectId);
var isColorValid = TryGet<int>(rowData, header, "Color", out var color);
var isDateCreatedValid = TryGet<DateTime>(rowData, header, "DateCreatedUtc", out var dateCreatedUtc);
var isDateModifiedValid = TryGet<DateTime>(rowData, header, "DateModifiedUtc", out var dateModifiedUtc);
if (!isColorValid || !isDateCreatedValid || !isDateModifiedValid)
continue;
var name = GetQuoted(GetValueFromHeader(rowData, header, "Name"))?.Trim();

if (!string.IsNullOrEmpty(name) && await _storageProvider.GetProjectAsync(name, userContext) == null)
{
var project = new Project
{
Name = name,
Description = GetQuoted(GetValueFromHeader(rowData, header, "Description")),
Location = GetQuoted(GetValueFromHeader(rowData, header, "Location")),
Color = color,
DateCreatedUtc = dateCreatedUtc
};
try
{
project = await _storageProvider.AddProjectAsync(project, userContext);
_temporaryKeyTracker.AddKeyMapping("Projects", "ProjectId", projectId, project.ProjectId);
result.TotalRowsImported++;
result.RowsImportedByTable["Projects"]++;
}
catch (Exception ex)
{
result.Errors.Add($"[Row {rowNumber}, Project with name '{name}' could not be added. Error: {ex.Message}");
}
}
else
{
result.Warnings.Add($"[Row {rowNumber}] Project with name '{name}' already exists.");
}
}
break;
case "parttypes":
{
// import partTypes info
var isPartTypeIdValid = TryGet<long>(rowData, header, "PartTypeId", out var partTypeId);
var isParentPartTypeIdValid = TryGet<long?>(rowData, header, "ParentPartTypeId", out var parentPartTypeId);
var isDateCreatedValid = TryGet<DateTime>(rowData, header, "DateCreatedUtc", out var dateCreatedUtc);
if (!isParentPartTypeIdValid || !isDateCreatedValid)
continue;
// import BOM info
var isPartNumberValid = TryGet<string?>(rowData, header, "MPN", out var partNumber);
var isQuantityValid = TryGet<int>(rowData, header, "Qty", out var quantity);
var isReferenceValid = TryGet<string?>(rowData, header, "Reference", out var reference);
var isNoteValid = TryGet<string?>(rowData, header, "Value", out var note);

if (!isPartNumberValid || !isQuantityValid || !isReferenceValid)
continue;

ProjectPartAssignment assignment = new ProjectPartAssignment();
assignment.ProjectId = project.ProjectId;
assignment.Quantity = quantity;
assignment.Notes = note;
assignment.SchematicReferenceId = reference;

var name = GetQuoted(GetValueFromHeader(rowData, header, "Name"))?.Trim();
// part types need to have a unique name for the user and can not be part of global part types
if (!string.IsNullOrEmpty(name) && !partTypes.Any(x => x.Name?.Equals(name, StringComparison.InvariantCultureIgnoreCase) == true))
{
if (parentPartTypeId == 0)
parentPartTypeId = null;
var partType = new PartType
{
ParentPartTypeId = parentPartTypeId != null ? _temporaryKeyTracker.GetMappedId("PartTypes", "PartTypeId", parentPartTypeId.Value) : null,
Name = name,
DateCreatedUtc = dateCreatedUtc
};
partType = await _storageProvider.GetOrCreatePartTypeAsync(partType, userContext);
if (partType != null)
{
_temporaryKeyTracker.AddKeyMapping("PartTypes", "PartTypeId", partTypeId,
partType.PartTypeId);
result.TotalRowsImported++;
result.RowsImportedByTable["PartTypes"]++;
}
}
else
{
result.Warnings.Add($"[Row {rowNumber}] PartType with name '{name}' already exists.");
}
}
break;
case "parts":
{
// import parts info
var isPartIdValid = TryGet<long>(rowData, header, "PartId", out var partId);
var isPartTypeIdValid = TryGet<long>(rowData, header, "PartTypeId", out var partTypeId);
var isBinNumberValid = TryGet<string?>(rowData, header, "BinNumber", out var binNumber);
var isBinNumber2Valid = TryGet<string?>(rowData, header, "BinNumber2", out var binNumber2);
var isCostValid = TryGet<double>(rowData, header, "Cost", out var cost);
var isDatasheetUrlValid = TryGet<string?>(rowData, header, "DatasheetUrl", out var datasheetUrl);
var isDescriptionValid = TryGet<string?>(rowData, header, "Description", out var description);
var isDigiKeyPartNumberValid = TryGet<string?>(rowData, header, "DigiKeyPartNumber", out var digiKeyPartNumber);
var isImageUrlValid = TryGet<string?>(rowData, header, "ImageUrl", out var imageUrl);
var isKeywordsValid = TryGet<string?>(rowData, header, "Keywords", out var keywords);
var isLocationValid = TryGet<string?>(rowData, header, "Location", out var location);
var isLowestCostSupplierValid = TryGet<string?>(rowData, header, "LowestCostSupplier", out var lowestCostSupplier);
var isLowestCostSupplierUrlValid = TryGet<string?>(rowData, header, "LowestCostSupplierUrl", out var lowestCostSupplierUrl);
var isLowStockThresholdValid = TryGet<int>(rowData, header, "LowStockThreshold", out var lowStockThreshold);
var isManufacturerValid = TryGet<string?>(rowData, header, "Manufacturer", out var manufacturer);
var isManufacturerPartNumberValid = TryGet<string?>(rowData, header, "ManufacturerPartNumber", out var manufacturerPartNumber);
var isMountingTypeIdValid = TryGet<int>(rowData, header, "MountingTypeId", out var mountingTypeId);
var isMouserPartNumberValid = TryGet<string?>(rowData, header, "MouserPartNumber", out var mouserPartNumber);
var isPackageTypeValid = TryGet<string?>(rowData, header, "PackageType", out var packageType);
var isPartNumberValid = TryGet<string?>(rowData, header, "PartNumber", out var partNumber);
var isProductUrlValid = TryGet<string?>(rowData, header, "ProductUrl", out var productUrl);
var isProjectIdValid = TryGet<long?>(rowData, header, "ProjectId", out var projectId);
var isQuantityValid = TryGet<long>(rowData, header, "Quantity", out var quantity);
var isSwarmPartNumberManufacturerIdValid = TryGet<int?>(rowData, header, "SwarmPartNumberManufacturerId", out var swarmPartNumberManufacturerId);
var isDateCreatedValid = TryGet<DateTime>(rowData, header, "DateCreatedUtc", out var dateCreatedUtc);
var part = await _storageProvider.GetPartAsync(partNumber, userContext);
if (part != null)
{
assignment.PartName = part.PartNumber;
assignment.PartId = part.PartId;
}
else
{
part = new Part();
part.PartNumber = partNumber;
part.Quantity = 0;
part.UserId = userContext?.UserId;
part.PartTypeId = (long)SystemDefaults.DefaultPartTypes.Other;
try
{
part = await _storageProvider.AddPartAsync(part, userContext);
}
catch (Exception ex)
{
// failed to add part
result.Errors.Add($"[Row {rowNumber}'] Part with PartNumber '{partNumber}' could not be added. Error: {ex.Message}");
}

if (!isPartTypeIdValid || !isBinNumberValid || !isBinNumber2Valid || !isCostValid || !isDatasheetUrlValid
|| !isDescriptionValid || !isDigiKeyPartNumberValid || !isImageUrlValid || !isKeywordsValid || !isLocationValid || !isLowestCostSupplierValid
|| !isLowestCostSupplierUrlValid || !isLowStockThresholdValid || !isManufacturerValid || !isManufacturerPartNumberValid || !isMountingTypeIdValid || !isMouserPartNumberValid
|| !isPackageTypeValid || !isPartNumberValid || !isProductUrlValid || !isProjectIdValid || !isQuantityValid || !isSwarmPartNumberManufacturerIdValid)
continue;
assignment.PartName = partNumber;
assignment.PartId = part.PartId;
}

if (!string.IsNullOrEmpty(partNumber) && await _storageProvider.GetPartAsync(partNumber, userContext) == null)
{
var part = new Part
{
PartTypeId = _temporaryKeyTracker.GetMappedId("PartTypes", "PartTypeId", partTypeId),
BinNumber = binNumber,
BinNumber2 = binNumber2,
Cost = cost,
DatasheetUrl = datasheetUrl,
Description = description,
DigiKeyPartNumber = digiKeyPartNumber,
ImageUrl = imageUrl,
Keywords = !string.IsNullOrEmpty(keywords) ? keywords.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries) : null,
Location = location,
LowestCostSupplier = lowestCostSupplier,
LowestCostSupplierUrl = lowestCostSupplierUrl,
LowStockThreshold = lowStockThreshold,
Manufacturer = manufacturer,
ManufacturerPartNumber = manufacturerPartNumber,
MountingTypeId = mountingTypeId,
MouserPartNumber = mouserPartNumber,
PackageType = packageType,
PartNumber = partNumber,
ProductUrl = productUrl,
ProjectId = projectId != null ? _temporaryKeyTracker.GetMappedId("Projects", "ProjectId", projectId.Value) : null,
Quantity = quantity,
//SwarmPartNumberManufacturerId = swarmPartNumberManufacturerId,
DateCreatedUtc = dateCreatedUtc
};
// some data validation required
if (part.ProjectId == 0) part.ProjectId = null;
if (part.UserId == 0) part.UserId = userContext?.UserId;
if (part.PartTypeId == 0) part.PartTypeId = (long)SystemDefaults.DefaultPartTypes.Other;
try
{
part = await _storageProvider.AddPartAsync(part, userContext);
_temporaryKeyTracker.AddKeyMapping("Parts", "PartId", partId, part.PartId);
result.TotalRowsImported++;
result.RowsImportedByTable["Parts"]++;
}
catch (Exception ex)
{
// failed to add part
result.Errors.Add($"[Row {rowNumber}, Part with PartNumber '{partNumber}' could not be added. Error: {ex.Message}");
}
}
else
{
result.Warnings.Add($"[Row {rowNumber}] Part with PartNumber '{partNumber}' already exists.");
}
}
break;
try
{
await _storageProvider.AddProjectPartAssignmentAsync(assignment, userContext);
}
catch (Exception ex)
{
result.Errors.Add($"[Row {rowNumber}] BOM entry '{partNumber}' could not be added. Error: {ex.Message}");
}

rowNumber++;
}
}
catch (Exception ex)
{
result.Errors.Add(ex.Message);
}
*/

result.Success = !result.Errors.Any();
return result;
}
Expand Down