feat: XLSX request list upload with smart column detection
- Add excelize v2 for XLSX parsing - Auto-detect column indices from header row using common DD checklist naming conventions (section/category/topic, item#/ref/no, description/ request/document, priority/urgency/criticality) - Detect file type by extension + PK magic bytes (xlsx is a zip) - Smart priority mapping: critical/urgent→high, nice/optional→low - Fall back to positional parsing (col 0=section, 1=item#, 2=desc) if headers don't match known patterns - Update modal label: 'Excel .xlsx or CSV — columns auto-detected'
This commit is contained in:
James
parent
13effdce40
commit
bdd1d9fdde
12
go.mod
12
go.mod
|
|
@ -8,4 +8,14 @@ require (
|
|||
github.com/mattn/go-sqlite3 v1.14.18
|
||||
)
|
||||
|
||||
require golang.org/x/crypto v0.48.0 // indirect
|
||||
require (
|
||||
github.com/richardlehane/mscfb v1.0.6 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.6 // indirect
|
||||
github.com/tiendc/go-deepcopy v1.7.2 // indirect
|
||||
github.com/xuri/efp v0.0.1 // indirect
|
||||
github.com/xuri/excelize/v2 v2.10.1 // indirect
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
)
|
||||
|
|
|
|||
16
go.sum
16
go.sum
|
|
@ -6,5 +6,21 @@ github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE
|
|||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
|
||||
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
|
||||
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
|
||||
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
|
||||
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
|
||||
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
|
||||
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
|
||||
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
|
||||
github.com/xuri/excelize/v2 v2.10.1 h1:V62UlqopMqha3kOpnlHy2CcRVw1V8E63jFoWUmMzxN0=
|
||||
github.com/xuri/excelize/v2 v2.10.1/go.mod h1:iG5tARpgaEeIhTqt3/fgXCGoBRt4hNXgCp3tfXKoOIc=
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
|
||||
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package handler
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -10,6 +11,7 @@ import (
|
|||
|
||||
"dealroom/internal/rbac"
|
||||
"dealroom/templates"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
func (h *Handler) handleRequestList(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -73,58 +75,133 @@ func (h *Handler) handleRequestListUpload(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
file, _, err := r.FormFile("request_list")
|
||||
file, header, err := r.FormFile("request_list")
|
||||
if err != nil {
|
||||
http.Error(w, "File is required", 400)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Parse CSV (supports basic CSV format: section, item_number, description, priority)
|
||||
reader := csv.NewReader(bufio.NewReader(file))
|
||||
reader.FieldsPerRecord = -1 // variable fields
|
||||
reader.TrimLeadingSpace = true
|
||||
|
||||
var items []struct {
|
||||
section, itemNumber, description, priority string
|
||||
// Read entire file into memory so we can detect type and re-read if needed
|
||||
raw, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading file", 500)
|
||||
return
|
||||
}
|
||||
|
||||
lineNum := 0
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
// Detect XLSX by filename extension or magic bytes (PK = zip/xlsx)
|
||||
fname := strings.ToLower(header.Filename)
|
||||
isXLSX := strings.HasSuffix(fname, ".xlsx") || strings.HasSuffix(fname, ".xls") ||
|
||||
(len(raw) >= 2 && raw[0] == 'P' && raw[1] == 'K')
|
||||
|
||||
type reqRow struct {
|
||||
section, itemNumber, description, priority string
|
||||
}
|
||||
var rows [][]string
|
||||
|
||||
if isXLSX {
|
||||
// Parse XLSX with excelize
|
||||
xf, err := excelize.OpenReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
continue
|
||||
http.Error(w, "Error parsing XLSX: "+err.Error(), 400)
|
||||
return
|
||||
}
|
||||
lineNum++
|
||||
if lineNum == 1 {
|
||||
// skip header row
|
||||
continue
|
||||
sheetName := xf.GetSheetName(0)
|
||||
xlRows, err := xf.GetRows(sheetName)
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading sheet: "+err.Error(), 400)
|
||||
return
|
||||
}
|
||||
if len(record) < 3 {
|
||||
continue
|
||||
rows = xlRows
|
||||
} else {
|
||||
// Parse CSV
|
||||
reader := csv.NewReader(bufio.NewReader(bytes.NewReader(raw)))
|
||||
reader.FieldsPerRecord = -1
|
||||
reader.TrimLeadingSpace = true
|
||||
csvRows, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
http.Error(w, "Error parsing CSV: "+err.Error(), 400)
|
||||
return
|
||||
}
|
||||
item := struct {
|
||||
section, itemNumber, description, priority string
|
||||
}{
|
||||
section: strings.TrimSpace(record[0]),
|
||||
itemNumber: strings.TrimSpace(record[1]),
|
||||
description: strings.TrimSpace(record[2]),
|
||||
priority: "medium",
|
||||
}
|
||||
if len(record) >= 4 {
|
||||
p := strings.ToLower(strings.TrimSpace(record[3]))
|
||||
if p == "high" || p == "medium" || p == "low" {
|
||||
item.priority = p
|
||||
rows = csvRows
|
||||
}
|
||||
|
||||
// Detect column indices from header row using common DD checklist naming conventions
|
||||
// Falls back to positional (col 0=section, 1=item#, 2=description, 3=priority)
|
||||
idxSection := 0
|
||||
idxItem := 1
|
||||
idxDesc := 2
|
||||
idxPriority := -1
|
||||
|
||||
if len(rows) > 0 {
|
||||
for ci, cell := range rows[0] {
|
||||
h := strings.ToLower(strings.TrimSpace(cell))
|
||||
switch {
|
||||
case contains(h, "section", "category", "topic", "area", "phase", "workstream"):
|
||||
idxSection = ci
|
||||
case contains(h, "item #", "item#", "item no", "no.", "ref", "number", "#"):
|
||||
idxItem = ci
|
||||
case contains(h, "description", "request", "document", "information", "detail", "item") && ci != idxSection:
|
||||
idxDesc = ci
|
||||
case contains(h, "priority", "urgency", "importance", "criticality"):
|
||||
idxPriority = ci
|
||||
}
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
var items []reqRow
|
||||
for ri, record := range rows {
|
||||
if ri == 0 {
|
||||
continue // skip header
|
||||
}
|
||||
if len(record) == 0 {
|
||||
continue
|
||||
}
|
||||
// Skip blank rows
|
||||
allBlank := true
|
||||
for _, c := range record {
|
||||
if strings.TrimSpace(c) != "" {
|
||||
allBlank = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allBlank {
|
||||
continue
|
||||
}
|
||||
|
||||
get := func(idx int) string {
|
||||
if idx >= 0 && idx < len(record) {
|
||||
return strings.TrimSpace(record[idx])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
desc := get(idxDesc)
|
||||
if desc == "" {
|
||||
continue // must have a description to be useful
|
||||
}
|
||||
|
||||
priority := "medium"
|
||||
if idxPriority >= 0 {
|
||||
p := strings.ToLower(get(idxPriority))
|
||||
switch {
|
||||
case strings.Contains(p, "high") || strings.Contains(p, "critical") || strings.Contains(p, "urgent"):
|
||||
priority = "high"
|
||||
case strings.Contains(p, "low") || strings.Contains(p, "nice") || strings.Contains(p, "optional"):
|
||||
priority = "low"
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, reqRow{
|
||||
section: get(idxSection),
|
||||
itemNumber: get(idxItem),
|
||||
description: desc,
|
||||
priority: priority,
|
||||
})
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
http.Error(w, "No valid items found in CSV", 400)
|
||||
http.Error(w, "No valid items found in file — check that the sheet has a header row and a description column", 400)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -233,6 +310,16 @@ func (h *Handler) autoAssignFilesToRequests(dealID string) {
|
|||
}
|
||||
}
|
||||
|
||||
// contains checks if s contains any of the given substrings.
|
||||
func contains(s string, subs ...string) bool {
|
||||
for _, sub := range subs {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *Handler) handleUpdateComment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
|
|
|
|||
|
|
@ -424,10 +424,10 @@ templ DealRoomDetail(profile *model.Profile, deal *model.Deal, folders []*model.
|
|||
<form action="/deals/requests/upload" method="POST" enctype="multipart/form-data" class="space-y-4">
|
||||
<input type="hidden" name="deal_id" value={ deal.ID }/>
|
||||
<div>
|
||||
<label class="block text-xs font-medium text-gray-400 mb-1">Request List File <span class="text-gray-600">(CSV: section, item_number, description, priority)</span></label>
|
||||
<label class="block text-xs font-medium text-gray-400 mb-1">Request List File <span class="text-gray-600">(Excel .xlsx or CSV — columns auto-detected)</span></label>
|
||||
<div class="border-2 border-dashed border-gray-700 rounded-lg p-4 text-center hover:border-teal-500/50 transition cursor-pointer" onclick="document.getElementById('reqListFile').click()">
|
||||
<svg class="w-6 h-6 text-gray-500 mx-auto mb-1" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"></path></svg>
|
||||
<p class="text-sm text-gray-400">Click to upload CSV</p>
|
||||
<p class="text-sm text-gray-400">Click to upload Excel or CSV</p>
|
||||
<p id="reqListFileName" class="text-xs text-teal-400 mt-1 hidden"></p>
|
||||
</div>
|
||||
<input type="file" id="reqListFile" name="request_list" accept=".csv,.xlsx,.xls" class="hidden" required onchange="if(this.files.length){var el=document.getElementById('reqListFileName');el.textContent=this.files[0].name;el.classList.remove('hidden')}"/>
|
||||
|
|
|
|||
Loading…
Reference in New Issue