inou/portal/upload.go

package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"inou/lib"
)

var processProgress sync.Map // dossierID → map[string]interface{}

// detectFileType identifies file type by magic bytes or data structure.
func detectFileType(data []byte) string {
	n := len(data)
	if n < 8 {
		return ""
	}
	// DICOM: 128-byte preamble + "DICM"
	if n > 132 && string(data[128:132]) == "DICM" {
		return "dicom"
	}
	// PNG: 8-byte signature \x89PNG\r\n\x1a\n
	if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4e && data[3] == 0x47 &&
		data[4] == 0x0d && data[5] == 0x0a && data[6] == 0x1a && data[7] == 0x0a {
		return "png"
	}
	// PDF: %PDF-N.N
	if n > 8 && string(data[:5]) == "%PDF-" && data[5] >= '1' && data[5] <= '9' && data[6] == '.' {
		return "pdf"
	}
	// ZIP: PK\x03\x04 (local file header)
	if data[0] == 0x50 && data[1] == 0x4b && data[2] == 0x03 && data[3] == 0x04 {
		return "zip"
	}
	// JPEG: \xff\xd8\xff (SOI + first marker)
	if data[0] == 0xff && data[1] == 0xd8 && data[2] == 0xff {
		return "jpg"
	}
	// Genome: tab-separated text with rs[0-9]+ in first data column
	if isGenomeData(data) {
		return "genetics"
	}
	// JSON: starts with { or [
	if b := firstNonSpace(data); b == '{' || b == '[' {
		return "json"
	}
	// Text: printable UTF-8 (fallback for markdown, CSV, plain text)
	if isReadableText(data) {
		return "text"
	}
	return ""
}

func firstNonSpace(data []byte) byte {
	for _, b := range data {
		if b != ' ' && b != '\t' && b != '\n' && b != '\r' {
			return b
		}
	}
	return 0
}

// isReadableText checks if content is printable UTF-8 text (not binary).
func isReadableText(data []byte) bool {
	peek := data
	if len(peek) > 1024 {
		peek = peek[:1024]
	}
	printable := 0
	for _, b := range peek {
		if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b < 127) {
			printable++
		} else if b >= 128 {
			printable++ // UTF-8 multibyte
		} else if b == 0 {
			return false // null byte = binary
		}
	}
	return printable > len(peek)*9/10 // >90% printable
}

var rsidRe = regexp.MustCompile(`^rs\d{1,12}\t`)

// isGenomeData checks whether text content has the structure of a genome raw data file:
// lines of tab-separated values where the first column is an rsID (rs[0-9]+).
// Requires at least 10 matching lines to avoid false positives.
func isGenomeData(data []byte) bool {
	peek := data
	if len(peek) > 8192 {
		peek = peek[:8192]
	}
	matches := 0
	for _, line := range strings.SplitN(string(peek), "\n", 100) {
		line = strings.TrimSpace(line)
		if line == "" || line[0] == '#' {
			continue
		}
		// Skip header lines
		if strings.HasPrefix(line, "rsid") || strings.HasPrefix(line, "RSID") {
			continue
		}
		if rsidRe.MatchString(line) {
			matches++
			if matches >= 10 {
				return true
			}
		} else {
			return false // non-matching data line = not genome
		}
	}
	return false
}
var sliceRe = regexp.MustCompile(`^(.+?) - (.+?) - slice (\d+)/(\d+)`)
var fileRe = regexp.MustCompile(`^file (\d+)/(\d+)`)

// Upload represents a file upload entry for display
type Upload struct {
	ID                                                              string
	FileName, FilePath, SizeHuman, UploadedAt, ExpiresAt, DeletedReason string
	Category, Status                                                string
	Deleted, CanUndo, CanReprocess                                  bool
}

// UploadData is the JSON structure stored in Entry.Data for uploads
type UploadData struct {
	Path       string `json:"path"`
	RelPath    string `json:"rel_path,omitempty"` // Original relative path for folder uploads
	Size       int64  `json:"size"`
	UploadedBy string `json:"uploaded_by"`
	Status     string `json:"status"`
}

func formatBytes(b int64) string {
	const unit = 1024
	if b < unit {
		return fmt.Sprintf("%d B", b)
	}
	div, exp := int64(unit), 0
	for n := b / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}

// getUploads returns all uploads for a dossier using lib.EntryList
func getUploads(dossierID string) []Upload {
	var uploads []Upload

	entries, err := lib.EntryList(lib.SystemAccessorID, "", lib.CategoryUpload, &lib.EntryFilter{ // nil ctx - internal operation
		DossierID: dossierID,
		Limit:     50,
	})
	if err != nil {
		return uploads
	}

	for _, e := range entries {
		var data map[string]interface{}
		json.Unmarshal([]byte(e.Data), &data)

		size, _ := data["size"].(float64)
		status, _ := data["status"].(string)
		path, _ := data["path"].(string)
		createdEntries, _ := data["created_entries"].([]interface{})

		u := Upload{
			ID:         e.EntryID,
			FileName:   e.Value,
			FilePath:   path,
			Category:   e.Type,
			Status:     status,
			SizeHuman:  formatBytes(int64(size)),
			UploadedAt: time.Unix(e.Timestamp, 0).UTC().Format("Jan 2"),
			ExpiresAt:  time.Unix(e.TimestampEnd, 0).UTC().Format("Jan 2"),
			CanUndo:        len(createdEntries) > 0,
		CanReprocess:   (e.Type == "pdf" || e.Type == "jpg" || e.Type == "png") && (status == "processed" || status == "failed"),
		}
		if e.Status != 0 {
			u.Deleted = true
			u.DeletedReason = "Deleted"
		}
		uploads = append(uploads, u)
	}
	return uploads
}

// getUploadEntry retrieves a single upload entry using lib.EntryGet
func getUploadEntry(entryID, dossierID string) (filePath, fileName, category, status string, deleted bool) {
	e, err := lib.EntryGet(nil, entryID) // nil ctx - internal operation
	if err != nil || e == nil {
		return
	}

	// Verify it belongs to the right dossier and is an upload
	if e.DossierID != dossierID || e.Category != lib.CategoryUpload {
		return
	}

	var data UploadData
	json.Unmarshal([]byte(e.Data), &data)

	fileName = e.Value
	category = e.Type
	filePath = data.Path
	status = data.Status
	deleted = e.Status != 0
	return
}

// findUploadByFilename finds existing uploads with the same filename
func findUploadByFilename(dossierID, filename string) []*lib.Entry {
	entries, err := lib.EntryList(lib.SystemAccessorID, "", lib.CategoryUpload, &lib.EntryFilter{
		DossierID: dossierID,
	})
	if err != nil {
		return nil
	}
	var matches []*lib.Entry
	for _, e := range entries {
		if e.Value == filename {
			matches = append(matches, e)
		}
	}
	return matches
}

func handleUploadPage(w http.ResponseWriter, r *http.Request) {
	p := getLoggedInDossier(r)
	if p == nil {
		http.Redirect(w, r, "/", http.StatusSeeOther)
		return
	}
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 4 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	target, _ := lib.DossierGet(p.DossierID, targetID)
	if target == nil {
		http.NotFound(w, r)
		return
	}
	lang := getLang(r)
	data := PageData{
		Page: "upload", Lang: lang, T: translations[lang],
		Dossier: p, TargetDossier: target,
		UploadList: getUploads(targetID),
	}
	render(w, r, data)
}

func handleUploadPost(w http.ResponseWriter, r *http.Request) {
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 4 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	r.ParseMultipartForm(10 << 30)
	file, header, err := r.FormFile("file")
	if err != nil {
		http.Error(w, "No file", http.StatusBadRequest)
		return
	}
	defer file.Close()

	relPath := r.FormValue("path")
	if relPath == "" {
		relPath = header.Filename
	}
	fileName := filepath.Base(relPath)
	category := r.FormValue("category")

	// Read file content
	content, err := io.ReadAll(file)
	if err != nil {
		http.Error(w, "Failed to read", http.StatusInternalServerError)
		return
	}

	// Auto-detect file type
	if category == "" {
		category = detectFileType(content)
	}

	// Generate file ID using lib.NewID() (Hex16 format)
	fileID := lib.NewID()

	// Store in uploads directory
	uploadDir := filepath.Join(uploadsDir, formatHexID(targetID))
	filePath := filepath.Join(uploadDir, fileID)
	os.MkdirAll(uploadDir, 0755)
	if err := lib.EncryptFile(content, filePath); err != nil {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`{"status":"error","message":"Upload failed, we've been notified"}`))
		return
	}
	written := int64(len(content))

	// Delete existing upload with same filename (re-upload cleanup)
	existingUploads := findUploadByFilename(targetID, fileName)
	if len(existingUploads) > 0 {
		log.Printf("[upload] Re-upload cleanup: %d existing for %q", len(existingUploads), fileName)
	}
	for _, old := range existingUploads {
		log.Printf("[upload] Deleting old upload %s (cat=%d type=%s)", old.EntryID, old.Category, old.Type)
		lib.EntryDelete("", targetID, &lib.Filter{EntryID: old.EntryID})
	}

	now := time.Now().Unix()
	expires := now + 7*24*60*60

	uploadData := UploadData{
		Path:       filePath,
		RelPath:    relPath,
		Size:       written,
		UploadedBy: p.DossierID,
		Status:     "uploaded",
	}
	dataJSON, _ := json.Marshal(uploadData)

	uploadEntry := &lib.Entry{
		DossierID:    targetID,
		Category:     lib.CategoryUpload,
		Type:         category,
		Value:        fileName,
		Timestamp:    now,
		TimestampEnd: expires,
		Data:         string(dataJSON),
	}
	if err := lib.EntryWrite("", uploadEntry); err != nil {
		log.Printf("[upload] ERROR writing upload entry for %s: %v", fileName, err)
	}
	entryID := uploadEntry.EntryID
	log.Printf("[upload] Created upload entry %s cat=%d type=%s file=%s", entryID, uploadEntry.Category, category, fileName)
	lib.AuditLog(p.DossierID, "file_upload", targetID, fileName)

	// Spawn processing for known types (JSON waits for batch processing like DICOM)
	if category == "genetics" {
		go processGenomeUpload(entryID, targetID, filePath)
	}
	if category == "pdf" || category == "jpg" || category == "png" {
		go processDocumentUpload(entryID, targetID, filePath, fileName)
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(fmt.Sprintf(`{"status":"ok","id":"%s"}`, entryID)))
}

func handleDeleteFile(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	// Parse /dossier/{hex}/files/{id}/delete
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 6 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]
	fileID := parts[4]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	// Get file info and created entries for cleanup
	entry, _ := lib.EntryGet(nil, fileID)
	fileName := ""
	if entry != nil && entry.DossierID == targetID {
		fileName = entry.Value
		var data map[string]interface{}
		json.Unmarshal([]byte(entry.Data), &data)
		if ids, _ := data["created_entries"].([]interface{}); len(ids) > 0 {
			for _, id := range ids {
				if parentID, _ := id.(string); parentID != "" {
					lib.EntryDelete("", targetID, &lib.Filter{EntryID: parentID})
				}
			}
		}
	}

	// Delete the upload entry and its file
	lib.EntryDelete("", targetID, &lib.Filter{EntryID: fileID})

	lib.AuditLog(p.DossierID, "file_delete", targetID, fileName)
	http.Redirect(w, r, fmt.Sprintf("/dossier/%s/upload", formatHexID(targetID)), http.StatusSeeOther)
}

func handleUpdateFile(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	// Parse /dossier/{hex}/files/{id}/update
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 6 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]
	fileID := parts[4]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	// Get entry using lib.EntryGet
	entry, err := lib.EntryGet(nil, fileID) // nil ctx - internal operation
	if err != nil || entry == nil || entry.DossierID != targetID || entry.Category != lib.CategoryUpload {
		http.NotFound(w, r)
		return
	}

	if entry.Status != 0 {
		http.NotFound(w, r) // Deleted
		return
	}

	var data UploadData
	json.Unmarshal([]byte(entry.Data), &data)

	if data.Status != "uploaded" {
		http.Error(w, "Cannot modify processed file", http.StatusBadRequest)
		return
	}

	newCategory := r.FormValue("category")
	if newCategory != "" && newCategory != entry.Type {
		entry.Type = newCategory
		lib.EntryWrite("", entry) // nil ctx - internal operation
		lib.AuditLog(p.DossierID, "file_category_change", targetID, entry.Value)
	}

	http.Redirect(w, r, fmt.Sprintf("/dossier/%s/upload", formatHexID(targetID)), http.StatusSeeOther)
}

func handleUndoImport(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 6 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]
	fileID := parts[4]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	entry, err := lib.EntryGet(nil, fileID)
	if err != nil || entry == nil || entry.DossierID != targetID || entry.Category != lib.CategoryUpload {
		http.NotFound(w, r)
		return
	}

	// Parse created_entries from upload data
	var data map[string]interface{}
	if json.Unmarshal([]byte(entry.Data), &data) != nil {
		http.Error(w, "Invalid upload data", http.StatusBadRequest)
		return
	}

	ids, _ := data["created_entries"].([]interface{})
	if len(ids) == 0 {
		http.Error(w, "No entries to undo", http.StatusBadRequest)
		return
	}

	// Delete each parent entry (cascading deletes children)
	for _, id := range ids {
		parentID, _ := id.(string)
		if parentID == "" {
			continue
		}
		lib.EntryDelete("", targetID, &lib.Filter{EntryID: parentID})
	}

	// Reset upload status to "uploaded" so it can be reprocessed
	data["status"] = "uploaded"
	delete(data, "created_entries")
	b, _ := json.Marshal(data)
	entry.Data = string(b)
	lib.EntryWrite("", entry)

	lib.AuditLog(p.DossierID, "undo_import", targetID, fmt.Sprintf("upload=%s orders=%d", entry.Value, len(ids)))

	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(`{"status":"ok"}`))
}

func handleReprocess(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 6 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]
	fileID := parts[4]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	entry, err := lib.EntryGet(nil, fileID)
	if err != nil || entry.DossierID != targetID || entry.Category != lib.CategoryUpload {
		http.Error(w, "Not found", http.StatusNotFound)
		return
	}

	var data map[string]interface{}
	json.Unmarshal([]byte(entry.Data), &data)

	// Delete previously created entries
	if ids, _ := data["created_entries"].([]interface{}); len(ids) > 0 {
		for _, id := range ids {
			if parentID, _ := id.(string); parentID != "" {
				lib.EntryDelete("", targetID, &lib.Filter{EntryID: parentID})
			}
		}
	}

	// Reset status and re-trigger
	data["status"] = "processing"
	delete(data, "created_entries")
	b, _ := json.Marshal(data)
	entry.Data = string(b)
	lib.EntryWrite("", entry)

	filePath, _ := data["path"].(string)
	go processDocumentUpload(fileID, targetID, filePath, entry.Value)

	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(`{"status":"ok"}`))
}

func handleProcessImaging(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 4 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]

	isSelf := targetID == p.DossierID
	if !isSelf {
		if access, found := getAccess(formatHexID(p.DossierID), formatHexID(targetID)); !found || !access.CanEdit {
			http.Error(w, "Forbidden", http.StatusForbidden)
			return
		}
	}

	// Don't start if already running
	if v, ok := processProgress.Load(targetID); ok {
		prog := v.(map[string]interface{})
		if prog["stage"] != "done" {
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(map[string]interface{}{"running": true})
			return
		}
	}

	processProgress.Store(targetID, map[string]interface{}{"stage": "starting"})
	go runProcessImaging(p.DossierID, targetID)

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{"started": true})
}

func handleProcessStatus(w http.ResponseWriter, r *http.Request) {
	p := getLoggedInDossier(r)
	if p == nil {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) < 4 {
		http.NotFound(w, r)
		return
	}
	targetID := parts[2]

	v, ok := processProgress.Load(targetID)
	if !ok {
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(map[string]interface{}{"stage": "idle"})
		return
	}
	prog := v.(map[string]interface{})
	if prog["stage"] == "done" {
		processProgress.Delete(targetID)
	}
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(prog)
}

func runProcessImaging(actorID, targetID string) {
	var processed int

	entries, err := lib.EntryList(lib.SystemAccessorID, "", lib.CategoryUpload, &lib.EntryFilter{DossierID: targetID})
	if err != nil {
		processProgress.Store(targetID, map[string]interface{}{"stage": "done", "error": err.Error()})
		return
	}

	// Split pending uploads into JSON and other (DICOM)
	result := &lib.ImportResult{}
	var pendingDICOM, pendingJSON []*lib.Entry
	for _, e := range entries {
		if e.Status != 0 || e.Type == "genetics" {
			continue
		}
		var d UploadData
		json.Unmarshal([]byte(e.Data), &d)
		if d.Status != "uploaded" {
			continue
		}
		switch e.Type {
		case "dicom", "imaging", "zip":
			pendingDICOM = append(pendingDICOM, e)
		default:
			pendingJSON = append(pendingJSON, e)
		}
	}

	if len(pendingDICOM) == 0 && len(pendingJSON) == 0 {
		processProgress.Store(targetID, map[string]interface{}{"stage": "done", "slices": 0})
		return
	}

	// --- Process DICOM files ---
	var importErr error
	if len(pendingDICOM) > 0 {
		totalFiles := len(pendingDICOM)
		logFn := func(format string, args ...interface{}) {
			msg := fmt.Sprintf(format, args...)
			if m := fileRe.FindStringSubmatch(msg); m != nil {
				processed, _ = strconv.Atoi(m[1])
			} else if m := sliceRe.FindStringSubmatch(msg); m != nil {
				processProgress.Store(targetID, map[string]interface{}{
					"stage": "importing", "study": m[1], "series": m[2],
					"processed": processed, "total": totalFiles,
				})
			}
		}

		processProgress.Store(targetID, map[string]interface{}{
			"stage": "decrypting", "total": totalFiles,
		})

		tempDir, err := os.MkdirTemp("", "dicom-import-*")
		if err != nil {
			processProgress.Store(targetID, map[string]interface{}{"stage": "done", "error": err.Error()})
			return
		}
		defer os.RemoveAll(tempDir)

		for i, e := range pendingDICOM {
			processProgress.Store(targetID, map[string]interface{}{
				"stage": "decrypting", "processed": i + 1, "total": totalFiles,
			})
			var d UploadData
			json.Unmarshal([]byte(e.Data), &d)
			relPath := d.RelPath
			if relPath == "" {
				relPath = e.Value
			}
			outPath := filepath.Join(tempDir, relPath)
			os.MkdirAll(filepath.Dir(outPath), 0755)
			content, err := lib.DecryptFile(d.Path)
			if err != nil {
				continue
			}
			os.WriteFile(outPath, content, 0644)
			d.Status = "processing"
			dataJSON, _ := json.Marshal(d)
			pendingDICOM[i].Data = string(dataJSON)
			lib.EntryWrite("", pendingDICOM[i])
		}

		result, importErr = lib.ImportDICOMFromPath(targetID, tempDir, "", logFn)

		finalStatus := "processed"
		if result.Slices == 0 && importErr == nil {
			finalStatus = "pending_importer"
		} else if importErr != nil {
			finalStatus = "failed"
		}
		for _, e := range pendingDICOM {
			var d UploadData
			json.Unmarshal([]byte(e.Data), &d)
			d.Status = finalStatus
			dataJSON, _ := json.Marshal(d)
			e.Data = string(dataJSON)
			if finalStatus == "processed" {
				e.Type = "imaging"
			}
			lib.EntryWrite("", e)
		}
	}

	// --- Process JSON files in batch ---
	var jsonImported int
	if len(pendingJSON) > 0 {
		processProgress.Store(targetID, map[string]interface{}{
			"stage": "importing_json", "processed": 0, "total": len(pendingJSON),
		})
		jsonImported = processJSONBatch(targetID, pendingJSON)
	}

	// Report results
	errStr := ""
	if importErr != nil {
		errStr = importErr.Error()
	}
	processProgress.Store(targetID, map[string]interface{}{
		"stage": "done", "studies": result.Studies,
		"series_count": result.Series, "slices": result.Slices,
		"json_imported": jsonImported,
		"error": errStr,
	})

	if importErr != nil {
		lib.AuditLog(actorID, "imaging_import", targetID, fmt.Sprintf("error=%v", importErr))
	} else if result.Slices > 0 {
		summary := fmt.Sprintf("studies=%d series=%d slices=%d", result.Studies, result.Series, result.Slices)
		lib.AuditLog(actorID, "imaging_import", targetID, summary)
		if target, _ := lib.DossierGet(actorID, targetID); target != nil {
			if actor, _ := lib.DossierGet("", actorID); actor != nil {
				lib.SendSignal(fmt.Sprintf("Import done: %s → %s (%s)", actor.Name, target.Name, summary))
			}
		}
	}
	if jsonImported > 0 {
		lib.AuditLog(actorID, "json_import", targetID, fmt.Sprintf("files=%d", jsonImported))
	}
}

// extractedEntry is the JSON structure returned by extraction prompts
type extractedEntry struct {
	Type              string                 `json:"type"`
	Value             string                 `json:"value"`
	Summary           string                 `json:"summary"`
	SummaryTranslated string                 `json:"summary_translated,omitempty"`
	SearchKey         string                 `json:"search_key,omitempty"`
	Timestamp         string                 `json:"timestamp,omitempty"`
	Data              map[string]interface{} `json:"data"`
	DataTranslated    map[string]interface{} `json:"data_translated,omitempty"`
	SourceSpans       []sourceSpan           `json:"source_spans,omitempty"`
	Results           []extractedEntry       `json:"results,omitempty"` // lab_order children
}

type sourceSpan struct {
	Start string `json:"start"`
	End   string `json:"end"`
}

// langName maps ISO 639-1 codes to full language names for LLM prompts.
func langName(code string) string {
	names := map[string]string{
		"en": "English", "de": "German", "ru": "Russian", "nl": "Dutch",
		"fr": "French", "es": "Spanish", "it": "Italian", "pt": "Portuguese",
		"ja": "Japanese", "zh": "Chinese", "ko": "Korean", "ar": "Arabic",
		"tr": "Turkish", "pl": "Polish", "uk": "Ukrainian",
	}
	if n, ok := names[code]; ok {
		return n
	}
	return code
}

// extractionSystem returns a system message for extraction prompts.
func extractionSystem(targetLang string) string {
	s := "You are a medical document extraction API. Return ONLY valid JSON (array or null). No explanations."
	if targetLang != "" {
		s += " For each entry, add summary_translated (" + targetLang + ") and data_translated (same keys, values in " + targetLang + ")."
	}
	return s
}

// loadExtractionPrompts discovers all extract_*.md files and returns {categoryID: prompt content}.
func loadExtractionPrompts() map[int]string {
	pattern := filepath.Join(lib.TrackerPromptsDir(), "extract_*.md")
	files, _ := filepath.Glob(pattern)
	prompts := make(map[int]string)
	for _, f := range files {
		// extract_device.md → "device"
		base := filepath.Base(f)
		name := strings.TrimPrefix(base, "extract_")
		name = strings.TrimSuffix(name, ".md")
		catID, ok := lib.CategoryFromString[name]
		if !ok {
			log.Printf("[doc-import] Unknown category in prompt file: %s", base)
			continue
		}
		data, err := os.ReadFile(f)
		if err != nil {
			continue
		}
		prompts[catID] = string(data)
	}
	return prompts
}

// parseTimestamp tries to parse a date string into Unix timestamp.
// Uses noon UTC to prevent date shift when displayed in negative-offset timezones.
func parseTimestamp(s string) int64 {
	if s == "" {
		return 0
	}
	for _, fmt := range []string{"2006-01-02", "02.01.2006", "01/02/2006", "Jan 2, 2006"} {
		if t, err := time.Parse(fmt, s); err == nil {
			return t.Unix()
		}
	}
	return 0
}

const (
	ocrModel        = "google/gemini-3-flash-preview"
	extractionModel = "stepfun/step-3.5-flash"
)

var ocrPrompt = `You are a medical document OCR system. Produce a faithful markdown transcription of this document.

The images are sequential pages of the same document. Process them in order: page 1 first, then page 2, etc.

Rules:
- Read each page top-to-bottom, left-to-right. For multi-column layouts, transcribe the full page as a human would read it.
- Preserve ALL text, dates, values, names, addresses, and structure
- Translate nothing — keep the original language
- Use markdown headers, lists, and formatting to reflect the document structure
- For tables, use markdown tables. Preserve numeric values exactly.
- Be complete — do not skip or summarize anything
- Do not describe visual elements (logos, signatures) — only transcribe text
- For handwritten text, transcribe as accurately as possible. Mark uncertain readings with [?]`

func processDocumentUpload(uploadID, dossierID, filePath, fileName string) {
	log.Printf("[doc-import] Starting for %s (%s)", fileName, dossierID)

	// Update upload status
	setUploadStatus := func(status string) {
		entry, err := lib.EntryGet(nil, uploadID)
		if err != nil {
			log.Printf("[doc-import] setUploadStatus(%s): EntryGet failed for %s: %v", status, uploadID, err)
			return
		}
		if entry == nil {
			log.Printf("[doc-import] setUploadStatus(%s): entry %s is nil (deleted?)", status, uploadID)
			return
		}
		log.Printf("[doc-import] setUploadStatus(%s): entry %s cat=%d type=%s", status, uploadID, entry.Category, entry.Type)
		var d UploadData
		json.Unmarshal([]byte(entry.Data), &d)
		d.Status = status
		data, _ := json.Marshal(d)
		entry.Data = string(data)
		lib.EntryWrite("", entry)
	}
	setUploadStatus("processing")

	// 1. Decrypt PDF
	pdfBytes, err := lib.DecryptFile(filePath)
	if err != nil {
		log.Printf("[doc-import] Decrypt failed: %v", err)
		setUploadStatus("failed")
		return
	}

	// 2. Convert to page images
	var pageFiles []string
	ext := strings.ToLower(filepath.Ext(fileName))
	if ext == ".jpg" || ext == ".jpeg" || ext == ".png" {
		// Image file: use directly as a single page
		tempDir, err := os.MkdirTemp("", "doc-import-*")
		if err != nil {
			log.Printf("[doc-import] MkdirTemp failed: %v", err)
			setUploadStatus("failed")
			return
		}
		defer os.RemoveAll(tempDir)
		imgPath := filepath.Join(tempDir, "page-1"+ext)
		if err := os.WriteFile(imgPath, pdfBytes, 0644); err != nil {
			log.Printf("[doc-import] WriteFile failed: %v", err)
			setUploadStatus("failed")
			return
		}
		pageFiles = []string{imgPath}
	} else {
		// PDF: convert to PNG pages via pdftoppm
		tempDir, err := os.MkdirTemp("", "doc-import-*")
		if err != nil {
			log.Printf("[doc-import] MkdirTemp failed: %v", err)
			setUploadStatus("failed")
			return
		}
		defer os.RemoveAll(tempDir)
		pdfPath := filepath.Join(tempDir, "input.pdf")
		if err := os.WriteFile(pdfPath, pdfBytes, 0644); err != nil {
			log.Printf("[doc-import] WriteFile failed: %v", err)
			setUploadStatus("failed")
			return
		}
		prefix := filepath.Join(tempDir, "page")
		cmd := exec.Command("pdftoppm", "-png", "-r", "200", pdfPath, prefix)
		if out, err := cmd.CombinedOutput(); err != nil {
			log.Printf("[doc-import] pdftoppm failed: %v: %s", err, out)
			setUploadStatus("failed")
			return
		}
		pageFiles, _ = filepath.Glob(prefix + "*.png")
		sort.Strings(pageFiles)
		if len(pageFiles) == 0 {
			log.Printf("[doc-import] No pages generated")
			setUploadStatus("failed")
			return
		}
	}
	log.Printf("[doc-import] %d pages to process", len(pageFiles))

	// 3. OCR: send pages to Fireworks vision model
	content := []interface{}{
		map[string]string{"type": "text", "text": ocrPrompt},
	}
	for _, pf := range pageFiles {
		imgBytes, err := os.ReadFile(pf)
		if err != nil {
			continue
		}
		b64 := base64.StdEncoding.EncodeToString(imgBytes)
		mime := "image/png"
		if strings.HasSuffix(strings.ToLower(pf), ".jpg") || strings.HasSuffix(strings.ToLower(pf), ".jpeg") {
			mime = "image/jpeg"
		}
		content = append(content, map[string]interface{}{
			"type": "image_url",
			"image_url": map[string]string{
				"url": "data:" + mime + ";base64," + b64,
			},
		})
	}

	messages := []map[string]interface{}{
		{"role": "user", "content": content},
	}

	markdown, err := lib.CallOpenRouter(ocrModel, messages, 16384)
	if err != nil {
		log.Printf("[doc-import] OCR failed: %v", err)
		setUploadStatus("failed")
		return
	}
	log.Printf("[doc-import] OCR done: %d chars markdown", len(markdown))

	// 4. Create document entry with markdown
	now := time.Now().Unix()
	docData := map[string]interface{}{
		"markdown":      markdown,
		"source_upload": uploadID,
		"pages":         len(pageFiles),
	}
	docDataJSON, _ := json.Marshal(docData)

	docType := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".")
	if docType == "jpeg" {
		docType = "jpg"
	}
	if docType == "" {
		docType = "pdf"
	}
	docEntry := &lib.Entry{
		DossierID: dossierID,
		Category:  lib.CategoryDocument,
		Type:      docType,
		Value:     fileName,
		Timestamp: now,
		Data:      string(docDataJSON),
	}
	lib.EntryWrite("", docEntry)
	docID := docEntry.EntryID
	log.Printf("[doc-import] Document entry created: %s", docID)

	// 5. Get dossier language
	var targetLang string
	if d, err := lib.DossierGet("", dossierID); err == nil && d.Preferences.Language != "" {
		targetLang = langName(d.Preferences.Language)
	}
	sysMsg := extractionSystem(targetLang)

	// 6. Translate markdown (blocking — we want this before extraction finishes)
	var translatedMarkdown string
	if targetLang != "" {
		prompt := fmt.Sprintf("Translate this medical document to %s. Preserve all markdown formatting, headers, tables, and structure exactly. Translate ALL text including headers and labels. Output ONLY the translated markdown, nothing else.\n\n%s", targetLang, markdown)
		msgs := []map[string]interface{}{{"role": "user", "content": prompt}}
		if resp, err := lib.CallOpenRouter(extractionModel, msgs, 16384); err != nil {
			log.Printf("[doc-import] Translation failed: %v", err)
		} else {
			translatedMarkdown = resp
			log.Printf("[doc-import] Translated to %s: %d chars", targetLang, len(resp))
		}
	}

	// 7. Fan out extraction — one goroutine per category prompt
	type catResult struct {
		Category int
		Entries  []extractedEntry
	}
	prompts := loadExtractionPrompts()
	log.Printf("[doc-import] Loaded %d extraction prompts", len(prompts))

	ch := make(chan catResult, len(prompts))
	var wg sync.WaitGroup
	for catID, promptTmpl := range prompts {
		wg.Add(1)
		go func(catID int, promptTmpl string) {
			defer wg.Done()
			prompt := strings.ReplaceAll(promptTmpl, "{{MARKDOWN}}", markdown)
			msgs := []map[string]interface{}{
				{"role": "system", "content": sysMsg},
				{"role": "user", "content": prompt},
			}
			resp, err := lib.CallOpenRouter(extractionModel, msgs, 8192)
			if err != nil {
				log.Printf("[doc-import] cat %d: API error: %v", catID, err)
				return
			}
			resp = strings.TrimSpace(resp)
			// Strip markdown code fences
			if strings.HasPrefix(resp, "```") {
				if nl := strings.Index(resp, "\n"); nl >= 0 {
					resp = resp[nl+1:]
				}
				if strings.HasSuffix(resp, "```") {
					resp = resp[:len(resp)-3]
				}
				resp = strings.TrimSpace(resp)
			}
			log.Printf("[doc-import] cat %d: %d chars response", catID, len(resp))
			if resp == "null" || resp == "[]" || resp == "" {
				return
			}
			var entries []extractedEntry
			if err := json.Unmarshal([]byte(resp), &entries); err != nil {
				var single extractedEntry
				if err2 := json.Unmarshal([]byte(resp), &single); err2 == nil && single.Summary != "" {
					entries = []extractedEntry{single}
				} else {
					log.Printf("[doc-import] cat %d: parse error: %v\nraw: %.300s", catID, err, resp)
					return
				}
			}
			if len(entries) > 0 {
				ch <- catResult{Category: catID, Entries: entries}
			}
		}(catID, promptTmpl)
	}
	wg.Wait()
	close(ch)

	var results []catResult
	for r := range ch {
		results = append(results, r)
	}

	// Save translated markdown
	if translatedMarkdown != "" {
		if de, err := lib.EntryGet(nil, docID); err == nil {
			var dd map[string]interface{}
			json.Unmarshal([]byte(de.Data), &dd)
			dd["markdown_translated"] = translatedMarkdown
			dd["translated_to"] = targetLang
			b, _ := json.Marshal(dd)
			de.Data = string(b)
			lib.EntryWrite("", de)
		}
	}

	totalEntries := 0
	for _, r := range results {
		totalEntries += len(r.Entries)
	}
	log.Printf("[doc-import] Extraction complete: %d categories, %d entries", len(results), totalEntries)

	// 8. Create entries
	var createdIDs []string
	createEntry := func(parentID string, category int, e extractedEntry) string {
		dataMap := map[string]interface{}{"source_doc_id": docID}
		for k, v := range e.Data {
			dataMap[k] = v
		}
		if len(e.SourceSpans) > 0 {
			dataMap["source_spans"] = e.SourceSpans
		}
		if e.SummaryTranslated != "" {
			dataMap["summary_translated"] = e.SummaryTranslated
		}
		if len(e.DataTranslated) > 0 {
			dataMap["data_translated"] = e.DataTranslated
		}
		dataJSON, _ := json.Marshal(dataMap)
		ts := parseTimestamp(e.Timestamp)
		entry := &lib.Entry{
			DossierID: dossierID, ParentID: parentID, Category: category,
			Type: e.Type, Value: e.Value, Summary: e.Summary, SearchKey: e.SearchKey,
			Timestamp: ts, Data: string(dataJSON),
		}
		lib.EntryWrite("", entry)
		createdIDs = append(createdIDs, entry.EntryID)
		return entry.EntryID
	}
	for _, r := range results {
		for _, e := range r.Entries {
			if e.Type == "lab_order" && len(e.Results) > 0 {
				orderID := createEntry(docID, r.Category, extractedEntry{
					Type: "lab_order", Value: e.Value, Timestamp: e.Timestamp, SourceSpans: e.SourceSpans,
					SummaryTranslated: e.SummaryTranslated, DataTranslated: e.DataTranslated,
				})
				for _, child := range e.Results {
					if child.Timestamp == "" {
						child.Timestamp = e.Timestamp
					}
					createEntry(orderID, r.Category, child)
				}
			} else {
				createEntry(docID, r.Category, e)
			}
		}
	}

	// Update document timestamp to earliest extracted date
	var earliest int64
	for _, r := range results {
		for _, e := range r.Entries {
			if ts := parseTimestamp(e.Timestamp); ts > 0 && (earliest == 0 || ts < earliest) {
				earliest = ts
			}
		}
	}
	if earliest > 0 {
		if de, err := lib.EntryGet(nil, docID); err == nil {
			de.Timestamp = earliest
			lib.EntryWrite("", de)
		}
	}

	// 9. Update upload status
	if entry, err := lib.EntryGet(nil, uploadID); err == nil {
		var data map[string]interface{}
		json.Unmarshal([]byte(entry.Data), &data)
		data["status"] = "processed"
		data["created_entries"] = append([]string{docID}, createdIDs...)
		b, _ := json.Marshal(data)
		entry.Data = string(b)
		lib.EntryWrite("", entry)
	}

	log.Printf("[doc-import] Complete: %s → doc=%s, %d extracts", fileName, docID, len(createdIDs))
	lib.AuditLog("", "doc_import", dossierID, fmt.Sprintf("file=%s doc=%s categories=%d", fileName, docID, len(results)))
}