Initial commit
This commit is contained in:
James
commit
880f9dab9d
|
|
@ -0,0 +1,5 @@
|
||||||
|
# Binaries
|
||||||
|
*.exe
|
||||||
|
node_modules/
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
.PHONY: build run install clean dev deps
|
||||||
|
|
||||||
|
BINARY=docman
|
||||||
|
VERSION=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||||
|
BUILD_DIR=build
|
||||||
|
|
||||||
|
# Build flags
|
||||||
|
LDFLAGS=-ldflags "-s -w -X main.Version=$(VERSION)"
|
||||||
|
CGO_CFLAGS=-DSQLITE_ENABLE_FTS5
|
||||||
|
TAGS=-tags "fts5"
|
||||||
|
|
||||||
|
build:
|
||||||
|
@mkdir -p $(BUILD_DIR)
|
||||||
|
CGO_CFLAGS="$(CGO_CFLAGS)" go build $(TAGS) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY) ./cmd/docman
|
||||||
|
|
||||||
|
run: build
|
||||||
|
./$(BUILD_DIR)/$(BINARY)
|
||||||
|
|
||||||
|
dev:
|
||||||
|
go run ./cmd/docman -port 8200
|
||||||
|
|
||||||
|
deps:
|
||||||
|
go mod download
|
||||||
|
go mod tidy
|
||||||
|
|
||||||
|
# Install to ~/bin
|
||||||
|
install: build
|
||||||
|
@mkdir -p $(HOME)/bin
|
||||||
|
cp $(BUILD_DIR)/$(BINARY) $(HOME)/bin/
|
||||||
|
@echo "Installed to $(HOME)/bin/$(BINARY)"
|
||||||
|
|
||||||
|
# Install systemd service
|
||||||
|
install-service:
|
||||||
|
@mkdir -p $(HOME)/.config/systemd/user
|
||||||
|
@envsubst < deploy/docman.service > $(HOME)/.config/systemd/user/docman.service
|
||||||
|
systemctl --user daemon-reload
|
||||||
|
systemctl --user enable docman
|
||||||
|
@echo "Service installed. Start with: systemctl --user start docman"
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf $(BUILD_DIR)
|
||||||
|
|
||||||
|
# Development helpers
|
||||||
|
test:
|
||||||
|
go test -v ./...
|
||||||
|
|
||||||
|
lint:
|
||||||
|
golangci-lint run
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
go fmt ./...
|
||||||
|
|
@ -0,0 +1,116 @@
|
||||||
|
# DocMan - Document Management System
|
||||||
|
|
||||||
|
AI-powered document scanning, OCR, classification, and search.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run directly (dev mode)
|
||||||
|
cd ~/dev/docman
|
||||||
|
make dev
|
||||||
|
|
||||||
|
# Or run the installed binary
|
||||||
|
~/bin/docman -port 8200
|
||||||
|
```
|
||||||
|
|
||||||
|
Open http://localhost:8200
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Auto-processing**: Drop PDFs/images into `~/documents/inbox/` → auto-classified and indexed
|
||||||
|
- **PDF Preview**: Built-in PDF viewer
|
||||||
|
- **Full-text Search**: FTS5 + semantic search
|
||||||
|
- **Categories**: taxes, expenses, bills, medical, contacts, legal, insurance, banking, receipts
|
||||||
|
- **Expense Tracking**: Filter by date, export to CSV, track tax-deductible items
|
||||||
|
- **Markdown Records**: Each document gets a searchable markdown file
|
||||||
|
|
||||||
|
## Scanner Setup
|
||||||
|
|
||||||
|
1. **Set scanner to save to SMB share:**
|
||||||
|
- Share: `\\192.168.1.16\documents\inbox` (or wherever james is)
|
||||||
|
- Or use your scanner's app to save to `~/documents/inbox/`
|
||||||
|
|
||||||
|
2. **Workflow:**
|
||||||
|
- Scan document → lands in inbox
|
||||||
|
- DocMan auto-processes (OCR → classify → store)
|
||||||
|
- View/search in web UI
|
||||||
|
|
||||||
|
## Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
~/documents/
|
||||||
|
├── inbox/ # Drop scans here (auto-processed)
|
||||||
|
├── store/ # PDF storage (by checksum)
|
||||||
|
├── records/ # Markdown records by category
|
||||||
|
│ ├── taxes/
|
||||||
|
│ ├── expenses/
|
||||||
|
│ ├── bills/
|
||||||
|
│ └── ...
|
||||||
|
└── index/ # SQLite database
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
```bash
|
||||||
|
FIREWORKS_API_KEY=fw_xxx # Required for AI classification
|
||||||
|
```
|
||||||
|
|
||||||
|
### Command Line Options
|
||||||
|
|
||||||
|
```
|
||||||
|
-port HTTP port (default: 8200)
|
||||||
|
-data Data directory (default: ~/documents)
|
||||||
|
-ai-endpoint AI API endpoint (default: Fireworks)
|
||||||
|
-ai-key AI API key
|
||||||
|
-ai-model Classification model
|
||||||
|
-embed-model Embedding model
|
||||||
|
-watch Only watch inbox, don't start web server
|
||||||
|
```
|
||||||
|
|
||||||
|
## Systemd Service
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Edit to add your Fireworks API key
|
||||||
|
nano ~/.config/systemd/user/docman.service
|
||||||
|
|
||||||
|
# Enable and start
|
||||||
|
systemctl --user daemon-reload
|
||||||
|
systemctl --user enable docman
|
||||||
|
systemctl --user start docman
|
||||||
|
|
||||||
|
# Check status
|
||||||
|
systemctl --user status docman
|
||||||
|
journalctl --user -u docman -f
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| GET | `/` | Dashboard |
|
||||||
|
| GET | `/browse` | Browse documents |
|
||||||
|
| GET | `/doc/:id` | View document |
|
||||||
|
| GET | `/search` | Search page |
|
||||||
|
| GET | `/expenses` | Expenses tracker |
|
||||||
|
| GET | `/upload` | Upload page |
|
||||||
|
| POST | `/api/upload` | Upload document |
|
||||||
|
| GET | `/api/documents/:id` | Get document JSON |
|
||||||
|
| PATCH | `/api/documents/:id` | Update document |
|
||||||
|
| DELETE | `/api/documents/:id` | Delete document |
|
||||||
|
| GET | `/api/search?q=` | Search API |
|
||||||
|
| GET | `/api/expenses/export` | Export CSV |
|
||||||
|
| GET | `/api/stats` | Dashboard stats |
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
System packages (already installed):
|
||||||
|
- `poppler-utils` (pdftotext, pdfinfo, pdftoppm)
|
||||||
|
- `tesseract-ocr` (OCR for scanned images)
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Without FIREWORKS_API_KEY, documents will be categorized as "uncategorized"
|
||||||
|
- The inbox watcher runs continuously, processing new files automatically
|
||||||
|
- Markdown files are searchable even without embeddings
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,164 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"docman/internal/db"
|
||||||
|
"docman/internal/handlers"
|
||||||
|
"docman/internal/processor"
|
||||||
|
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
|
"github.com/labstack/echo/v4/middleware"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
var (
|
||||||
|
port = flag.Int("port", 8200, "HTTP port")
|
||||||
|
dataDir = flag.String("data", "", "Data directory (default: ~/documents)")
|
||||||
|
aiEndpoint = flag.String("ai-endpoint", "https://api.fireworks.ai/inference/v1", "AI API endpoint")
|
||||||
|
aiKey = flag.String("ai-key", "", "AI API key (or FIREWORKS_API_KEY env)")
|
||||||
|
aiModel = flag.String("ai-model", "accounts/fireworks/models/qwen2-vl-72b-instruct", "AI model for classification")
|
||||||
|
embedModel = flag.String("embed-model", "nomic-ai/nomic-embed-text-v1.5", "Embedding model")
|
||||||
|
watchOnly = flag.Bool("watch", false, "Only watch inbox, don't start web server")
|
||||||
|
)
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// Resolve data directory
|
||||||
|
if *dataDir == "" {
|
||||||
|
home, _ := os.UserHomeDir()
|
||||||
|
*dataDir = filepath.Join(home, "documents")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve AI key
|
||||||
|
if *aiKey == "" {
|
||||||
|
*aiKey = os.Getenv("FIREWORKS_API_KEY")
|
||||||
|
}
|
||||||
|
if *aiKey == "" {
|
||||||
|
log.Println("Warning: No AI API key provided. Classification will fail.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create directories
|
||||||
|
dirs := map[string]string{
|
||||||
|
"inbox": filepath.Join(*dataDir, "inbox"),
|
||||||
|
"store": filepath.Join(*dataDir, "store"),
|
||||||
|
"records": filepath.Join(*dataDir, "records"),
|
||||||
|
"index": filepath.Join(*dataDir, "index"),
|
||||||
|
}
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||||
|
log.Fatalf("Failed to create directory %s: %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open database
|
||||||
|
dbPath := filepath.Join(dirs["index"], "docman.db")
|
||||||
|
database, err := db.Open(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to open database: %v", err)
|
||||||
|
}
|
||||||
|
defer database.Close()
|
||||||
|
|
||||||
|
if err := database.Init(); err != nil {
|
||||||
|
log.Fatalf("Failed to initialize database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create processor
|
||||||
|
proc := processor.New(processor.Config{
|
||||||
|
InboxDir: dirs["inbox"],
|
||||||
|
StoreDir: dirs["store"],
|
||||||
|
RecordsDir: dirs["records"],
|
||||||
|
AIEndpoint: *aiEndpoint,
|
||||||
|
AIKey: *aiKey,
|
||||||
|
AIModel: *aiModel,
|
||||||
|
EmbedModel: *embedModel,
|
||||||
|
}, database)
|
||||||
|
|
||||||
|
// Context for graceful shutdown
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Start inbox watcher
|
||||||
|
go func() {
|
||||||
|
if err := proc.Watch(ctx); err != nil && err != context.Canceled {
|
||||||
|
log.Printf("Watcher error: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if *watchOnly {
|
||||||
|
log.Printf("Watching inbox: %s", dirs["inbox"])
|
||||||
|
// Wait for signal
|
||||||
|
sigCh := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
<-sigCh
|
||||||
|
cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create Echo server
|
||||||
|
e := echo.New()
|
||||||
|
e.HideBanner = true
|
||||||
|
|
||||||
|
// Middleware
|
||||||
|
e.Use(middleware.Logger())
|
||||||
|
e.Use(middleware.Recover())
|
||||||
|
e.Use(middleware.CORS())
|
||||||
|
|
||||||
|
// Template renderer
|
||||||
|
e.Renderer = &templateRenderer{}
|
||||||
|
|
||||||
|
// Handlers
|
||||||
|
h := handlers.New(database, proc, dirs["store"], dirs["records"])
|
||||||
|
|
||||||
|
// Page routes
|
||||||
|
e.GET("/", h.Dashboard)
|
||||||
|
e.GET("/browse", h.Browse)
|
||||||
|
e.GET("/doc/:id", h.Document)
|
||||||
|
e.GET("/search", h.Search)
|
||||||
|
e.GET("/expenses", h.Expenses)
|
||||||
|
e.GET("/upload", h.Upload)
|
||||||
|
|
||||||
|
// API routes
|
||||||
|
e.POST("/api/upload", h.APIUpload)
|
||||||
|
e.GET("/api/documents/:id", h.APIDocument)
|
||||||
|
e.PATCH("/api/documents/:id", h.APIUpdateDocument)
|
||||||
|
e.DELETE("/api/documents/:id", h.APIDeleteDocument)
|
||||||
|
e.GET("/api/search", h.APISearch)
|
||||||
|
e.GET("/api/expenses/export", h.APIExportExpenses)
|
||||||
|
e.GET("/api/stats", h.APIStats)
|
||||||
|
|
||||||
|
// Static file serving
|
||||||
|
e.GET("/pdf/:filename", h.ServePDF)
|
||||||
|
e.GET("/markdown", h.ServeMarkdown)
|
||||||
|
|
||||||
|
// Graceful shutdown
|
||||||
|
go func() {
|
||||||
|
sigCh := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
<-sigCh
|
||||||
|
log.Println("Shutting down...")
|
||||||
|
cancel()
|
||||||
|
e.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Start server
|
||||||
|
log.Printf("DocMan starting on http://localhost:%d", *port)
|
||||||
|
log.Printf("Data directory: %s", *dataDir)
|
||||||
|
log.Printf("Inbox: %s", dirs["inbox"])
|
||||||
|
if err := e.Start(fmt.Sprintf(":%d", *port)); err != nil {
|
||||||
|
log.Printf("Server stopped: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type templateRenderer struct{}
|
||||||
|
|
||||||
|
func (t *templateRenderer) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
|
||||||
|
return handlers.NewTemplateRenderer().Render(w, name, data, c)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
[Unit]
|
||||||
|
Description=DocMan - Document Management System
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=${HOME}/bin/docman -port 8200 -ai-key ${FIREWORKS_API_KEY}
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
Environment=HOME=${HOME}
|
||||||
|
Environment=FIREWORKS_API_KEY=${FIREWORKS_API_KEY}
|
||||||
|
WorkingDirectory=${HOME}
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
module docman
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/fsnotify/fsnotify v1.7.0
|
||||||
|
github.com/google/uuid v1.6.0
|
||||||
|
github.com/labstack/echo/v4 v4.12.0
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.22
|
||||||
|
github.com/sashabaranov/go-openai v1.29.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||||
|
github.com/labstack/gommon v0.4.2 // indirect
|
||||||
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||||
|
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||||
|
golang.org/x/crypto v0.26.0 // indirect
|
||||||
|
golang.org/x/net v0.28.0 // indirect
|
||||||
|
golang.org/x/sys v0.23.0 // indirect
|
||||||
|
golang.org/x/text v0.17.0 // indirect
|
||||||
|
golang.org/x/time v0.5.0 // indirect
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||||
|
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||||
|
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||||
|
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
|
||||||
|
github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
|
||||||
|
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
|
||||||
|
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
|
||||||
|
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||||
|
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||||
|
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||||
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/sashabaranov/go-openai v1.29.0 h1:eBH6LSjtX4md5ImDCX8hNhHQvaRf22zujiERoQpsvLo=
|
||||||
|
github.com/sashabaranov/go-openai v1.29.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||||
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
|
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
|
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
|
||||||
|
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||||
|
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||||
|
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||||
|
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||||
|
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||||
|
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
|
||||||
|
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||||
|
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||||
|
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||||
|
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|
@ -0,0 +1,406 @@
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DB struct {
|
||||||
|
*sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
type Document struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
OriginalName string `json:"original_name"`
|
||||||
|
Category string `json:"category"`
|
||||||
|
Subcategory string `json:"subcategory,omitempty"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Date *time.Time `json:"date,omitempty"`
|
||||||
|
Vendor string `json:"vendor,omitempty"`
|
||||||
|
Amount *float64 `json:"amount,omitempty"`
|
||||||
|
Currency string `json:"currency,omitempty"`
|
||||||
|
TaxDeductible bool `json:"tax_deductible"`
|
||||||
|
OCRText string `json:"ocr_text"`
|
||||||
|
Metadata json.RawMessage `json:"metadata,omitempty"`
|
||||||
|
Embedding []byte `json:"embedding,omitempty"`
|
||||||
|
StoragePath string `json:"storage_path"`
|
||||||
|
MarkdownPath string `json:"markdown_path"`
|
||||||
|
PageCount int `json:"page_count"`
|
||||||
|
FileSize int64 `json:"file_size"`
|
||||||
|
MimeType string `json:"mime_type"`
|
||||||
|
Checksum string `json:"checksum"`
|
||||||
|
ProcessedAt time.Time `json:"processed_at"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SearchResult struct {
|
||||||
|
Document
|
||||||
|
Score float64 `json:"score"`
|
||||||
|
Snippet string `json:"snippet"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func Open(path string) (*DB, error) {
|
||||||
|
db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.Ping(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &DB{db}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Init() error {
|
||||||
|
schema := `
|
||||||
|
CREATE TABLE IF NOT EXISTS documents (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
original_name TEXT NOT NULL,
|
||||||
|
category TEXT NOT NULL DEFAULT 'uncategorized',
|
||||||
|
subcategory TEXT,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
date TEXT,
|
||||||
|
vendor TEXT,
|
||||||
|
amount REAL,
|
||||||
|
currency TEXT DEFAULT 'USD',
|
||||||
|
tax_deductible INTEGER DEFAULT 0,
|
||||||
|
ocr_text TEXT,
|
||||||
|
metadata TEXT,
|
||||||
|
embedding BLOB,
|
||||||
|
storage_path TEXT NOT NULL,
|
||||||
|
markdown_path TEXT,
|
||||||
|
page_count INTEGER DEFAULT 1,
|
||||||
|
file_size INTEGER,
|
||||||
|
mime_type TEXT,
|
||||||
|
checksum TEXT,
|
||||||
|
processed_at TEXT,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_documents_category ON documents(category);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_documents_date ON documents(date);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_documents_vendor ON documents(vendor);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_documents_amount ON documents(amount);
|
||||||
|
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
ocr_text,
|
||||||
|
vendor,
|
||||||
|
category,
|
||||||
|
content='documents',
|
||||||
|
content_rowid='rowid'
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
|
||||||
|
INSERT INTO documents_fts(id, title, ocr_text, vendor, category)
|
||||||
|
VALUES (new.id, new.title, new.ocr_text, new.vendor, new.category);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
|
||||||
|
INSERT INTO documents_fts(documents_fts, id, title, ocr_text, vendor, category)
|
||||||
|
VALUES ('delete', old.id, old.title, old.ocr_text, old.vendor, old.category);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
|
||||||
|
INSERT INTO documents_fts(documents_fts, id, title, ocr_text, vendor, category)
|
||||||
|
VALUES ('delete', old.id, old.title, old.ocr_text, old.vendor, old.category);
|
||||||
|
INSERT INTO documents_fts(id, title, ocr_text, vendor, category)
|
||||||
|
VALUES (new.id, new.title, new.ocr_text, new.vendor, new.category);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS settings (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT
|
||||||
|
);
|
||||||
|
`
|
||||||
|
_, err := db.Exec(schema)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) InsertDocument(doc *Document) error {
|
||||||
|
var dateStr *string
|
||||||
|
if doc.Date != nil {
|
||||||
|
s := doc.Date.Format("2006-01-02")
|
||||||
|
dateStr = &s
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := db.Exec(`
|
||||||
|
INSERT INTO documents (
|
||||||
|
id, filename, original_name, category, subcategory, title, date,
|
||||||
|
vendor, amount, currency, tax_deductible, ocr_text, metadata,
|
||||||
|
embedding, storage_path, markdown_path, page_count, file_size,
|
||||||
|
mime_type, checksum, processed_at, created_at, updated_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`, doc.ID, doc.Filename, doc.OriginalName, doc.Category, doc.Subcategory,
|
||||||
|
doc.Title, dateStr, doc.Vendor, doc.Amount, doc.Currency, doc.TaxDeductible,
|
||||||
|
doc.OCRText, doc.Metadata, doc.Embedding, doc.StoragePath, doc.MarkdownPath,
|
||||||
|
doc.PageCount, doc.FileSize, doc.MimeType, doc.Checksum,
|
||||||
|
doc.ProcessedAt.Format(time.RFC3339),
|
||||||
|
doc.CreatedAt.Format(time.RFC3339),
|
||||||
|
doc.UpdatedAt.Format(time.RFC3339))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) UpdateDocument(doc *Document) error {
|
||||||
|
var dateStr *string
|
||||||
|
if doc.Date != nil {
|
||||||
|
s := doc.Date.Format("2006-01-02")
|
||||||
|
dateStr = &s
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := db.Exec(`
|
||||||
|
UPDATE documents SET
|
||||||
|
category = ?, subcategory = ?, title = ?, date = ?,
|
||||||
|
vendor = ?, amount = ?, currency = ?, tax_deductible = ?,
|
||||||
|
metadata = ?, updated_at = ?
|
||||||
|
WHERE id = ?
|
||||||
|
`, doc.Category, doc.Subcategory, doc.Title, dateStr,
|
||||||
|
doc.Vendor, doc.Amount, doc.Currency, doc.TaxDeductible,
|
||||||
|
doc.Metadata, time.Now().Format(time.RFC3339), doc.ID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) GetDocument(id string) (*Document, error) {
|
||||||
|
row := db.QueryRow(`SELECT * FROM documents WHERE id = ?`, id)
|
||||||
|
return scanDocument(row)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) DeleteDocument(id string) error {
|
||||||
|
_, err := db.Exec(`DELETE FROM documents WHERE id = ?`, id)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) ListDocuments(category string, limit, offset int) ([]*Document, error) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if category != "" {
|
||||||
|
rows, err = db.Query(`
|
||||||
|
SELECT * FROM documents
|
||||||
|
WHERE category = ?
|
||||||
|
ORDER BY COALESCE(date, created_at) DESC
|
||||||
|
LIMIT ? OFFSET ?
|
||||||
|
`, category, limit, offset)
|
||||||
|
} else {
|
||||||
|
rows, err = db.Query(`
|
||||||
|
SELECT * FROM documents
|
||||||
|
ORDER BY COALESCE(date, created_at) DESC
|
||||||
|
LIMIT ? OFFSET ?
|
||||||
|
`, limit, offset)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
return scanDocuments(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) RecentDocuments(limit int) ([]*Document, error) {
|
||||||
|
rows, err := db.Query(`
|
||||||
|
SELECT * FROM documents
|
||||||
|
ORDER BY created_at DESC
|
||||||
|
LIMIT ?
|
||||||
|
`, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
return scanDocuments(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) SearchFTS(query string, limit int) ([]*SearchResult, error) {
|
||||||
|
rows, err := db.Query(`
|
||||||
|
SELECT d.*,
|
||||||
|
bm25(documents_fts) as score,
|
||||||
|
snippet(documents_fts, 2, '<mark>', '</mark>', '...', 32) as snippet
|
||||||
|
FROM documents_fts f
|
||||||
|
JOIN documents d ON f.id = d.id
|
||||||
|
WHERE documents_fts MATCH ?
|
||||||
|
ORDER BY bm25(documents_fts)
|
||||||
|
LIMIT ?
|
||||||
|
`, query, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var results []*SearchResult
|
||||||
|
for rows.Next() {
|
||||||
|
var doc Document
|
||||||
|
var dateStr, processedStr, createdStr, updatedStr sql.NullString
|
||||||
|
var score float64
|
||||||
|
var snippet string
|
||||||
|
|
||||||
|
err := rows.Scan(
|
||||||
|
&doc.ID, &doc.Filename, &doc.OriginalName, &doc.Category, &doc.Subcategory,
|
||||||
|
&doc.Title, &dateStr, &doc.Vendor, &doc.Amount, &doc.Currency, &doc.TaxDeductible,
|
||||||
|
&doc.OCRText, &doc.Metadata, &doc.Embedding, &doc.StoragePath, &doc.MarkdownPath,
|
||||||
|
&doc.PageCount, &doc.FileSize, &doc.MimeType, &doc.Checksum,
|
||||||
|
&processedStr, &createdStr, &updatedStr, &score, &snippet,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if dateStr.Valid {
|
||||||
|
t, _ := time.Parse("2006-01-02", dateStr.String)
|
||||||
|
doc.Date = &t
|
||||||
|
}
|
||||||
|
if createdStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, createdStr.String)
|
||||||
|
doc.CreatedAt = t
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, &SearchResult{Document: doc, Score: score, Snippet: snippet})
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) GetStats() (map[string]interface{}, error) {
|
||||||
|
stats := make(map[string]interface{})
|
||||||
|
|
||||||
|
// Total documents
|
||||||
|
var total int
|
||||||
|
db.QueryRow(`SELECT COUNT(*) FROM documents`).Scan(&total)
|
||||||
|
stats["total"] = total
|
||||||
|
|
||||||
|
// This month
|
||||||
|
var thisMonth int
|
||||||
|
db.QueryRow(`SELECT COUNT(*) FROM documents WHERE created_at >= date('now', 'start of month')`).Scan(&thisMonth)
|
||||||
|
stats["this_month"] = thisMonth
|
||||||
|
|
||||||
|
// Total size
|
||||||
|
var totalSize int64
|
||||||
|
db.QueryRow(`SELECT COALESCE(SUM(file_size), 0) FROM documents`).Scan(&totalSize)
|
||||||
|
stats["total_size"] = totalSize
|
||||||
|
|
||||||
|
// By category
|
||||||
|
rows, err := db.Query(`SELECT category, COUNT(*) FROM documents GROUP BY category`)
|
||||||
|
if err == nil {
|
||||||
|
categories := make(map[string]int)
|
||||||
|
for rows.Next() {
|
||||||
|
var cat string
|
||||||
|
var count int
|
||||||
|
rows.Scan(&cat, &count)
|
||||||
|
categories[cat] = count
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
stats["by_category"] = categories
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) GetExpenses(year int, month int) ([]*Document, error) {
|
||||||
|
query := `
|
||||||
|
SELECT * FROM documents
|
||||||
|
WHERE category = 'expenses' AND amount IS NOT NULL
|
||||||
|
`
|
||||||
|
args := []interface{}{}
|
||||||
|
|
||||||
|
if year > 0 {
|
||||||
|
query += ` AND strftime('%Y', date) = ?`
|
||||||
|
args = append(args, fmt.Sprintf("%04d", year))
|
||||||
|
}
|
||||||
|
if month > 0 {
|
||||||
|
query += ` AND strftime('%m', date) = ?`
|
||||||
|
args = append(args, fmt.Sprintf("%02d", month))
|
||||||
|
}
|
||||||
|
|
||||||
|
query += ` ORDER BY date DESC`
|
||||||
|
|
||||||
|
rows, err := db.Query(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
return scanDocuments(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanDocument(row *sql.Row) (*Document, error) {
|
||||||
|
var doc Document
|
||||||
|
var dateStr, processedStr, createdStr, updatedStr sql.NullString
|
||||||
|
|
||||||
|
err := row.Scan(
|
||||||
|
&doc.ID, &doc.Filename, &doc.OriginalName, &doc.Category, &doc.Subcategory,
|
||||||
|
&doc.Title, &dateStr, &doc.Vendor, &doc.Amount, &doc.Currency, &doc.TaxDeductible,
|
||||||
|
&doc.OCRText, &doc.Metadata, &doc.Embedding, &doc.StoragePath, &doc.MarkdownPath,
|
||||||
|
&doc.PageCount, &doc.FileSize, &doc.MimeType, &doc.Checksum,
|
||||||
|
&processedStr, &createdStr, &updatedStr,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if dateStr.Valid {
|
||||||
|
t, _ := time.Parse("2006-01-02", dateStr.String)
|
||||||
|
doc.Date = &t
|
||||||
|
}
|
||||||
|
if processedStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, processedStr.String)
|
||||||
|
doc.ProcessedAt = t
|
||||||
|
}
|
||||||
|
if createdStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, createdStr.String)
|
||||||
|
doc.CreatedAt = t
|
||||||
|
}
|
||||||
|
if updatedStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, updatedStr.String)
|
||||||
|
doc.UpdatedAt = t
|
||||||
|
}
|
||||||
|
|
||||||
|
return &doc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanDocuments(rows *sql.Rows) ([]*Document, error) {
|
||||||
|
var docs []*Document
|
||||||
|
for rows.Next() {
|
||||||
|
var doc Document
|
||||||
|
var dateStr, processedStr, createdStr, updatedStr sql.NullString
|
||||||
|
|
||||||
|
err := rows.Scan(
|
||||||
|
&doc.ID, &doc.Filename, &doc.OriginalName, &doc.Category, &doc.Subcategory,
|
||||||
|
&doc.Title, &dateStr, &doc.Vendor, &doc.Amount, &doc.Currency, &doc.TaxDeductible,
|
||||||
|
&doc.OCRText, &doc.Metadata, &doc.Embedding, &doc.StoragePath, &doc.MarkdownPath,
|
||||||
|
&doc.PageCount, &doc.FileSize, &doc.MimeType, &doc.Checksum,
|
||||||
|
&processedStr, &createdStr, &updatedStr,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if dateStr.Valid {
|
||||||
|
t, _ := time.Parse("2006-01-02", dateStr.String)
|
||||||
|
doc.Date = &t
|
||||||
|
}
|
||||||
|
if processedStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, processedStr.String)
|
||||||
|
doc.ProcessedAt = t
|
||||||
|
}
|
||||||
|
if createdStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, createdStr.String)
|
||||||
|
doc.CreatedAt = t
|
||||||
|
}
|
||||||
|
if updatedStr.Valid {
|
||||||
|
t, _ := time.Parse(time.RFC3339, updatedStr.String)
|
||||||
|
doc.UpdatedAt = t
|
||||||
|
}
|
||||||
|
|
||||||
|
docs = append(docs, &doc)
|
||||||
|
}
|
||||||
|
return docs, nil
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,700 @@
|
||||||
|
package processor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"docman/internal/db"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
openai "github.com/sashabaranov/go-openai"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Processor struct {
|
||||||
|
db *db.DB
|
||||||
|
inboxDir string
|
||||||
|
storeDir string
|
||||||
|
recordsDir string
|
||||||
|
aiClient *openai.Client
|
||||||
|
aiModel string
|
||||||
|
embedModel string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
InboxDir string
|
||||||
|
StoreDir string
|
||||||
|
RecordsDir string
|
||||||
|
AIEndpoint string // Fireworks API endpoint
|
||||||
|
AIKey string
|
||||||
|
AIModel string // e.g., "accounts/fireworks/models/qwen2-vl-72b-instruct"
|
||||||
|
EmbedModel string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Classification struct {
|
||||||
|
Category string `json:"category"`
|
||||||
|
Subcategory string `json:"subcategory,omitempty"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Date string `json:"date,omitempty"`
|
||||||
|
Vendor string `json:"vendor,omitempty"`
|
||||||
|
Amount *float64 `json:"amount,omitempty"`
|
||||||
|
Currency string `json:"currency,omitempty"`
|
||||||
|
TaxDeductible bool `json:"tax_deductible"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
KeyFields map[string]string `json:"key_fields,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(cfg Config, database *db.DB) *Processor {
|
||||||
|
config := openai.DefaultConfig(cfg.AIKey)
|
||||||
|
config.BaseURL = cfg.AIEndpoint
|
||||||
|
|
||||||
|
return &Processor{
|
||||||
|
db: database,
|
||||||
|
inboxDir: cfg.InboxDir,
|
||||||
|
storeDir: cfg.StoreDir,
|
||||||
|
recordsDir: cfg.RecordsDir,
|
||||||
|
aiClient: openai.NewClientWithConfig(config),
|
||||||
|
aiModel: cfg.AIModel,
|
||||||
|
embedModel: cfg.EmbedModel,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) Watch(ctx context.Context) error {
|
||||||
|
// Ensure directories exist
|
||||||
|
for _, dir := range []string{p.inboxDir, p.storeDir, p.recordsDir} {
|
||||||
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("create directory %s: %w", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process existing files first
|
||||||
|
entries, _ := os.ReadDir(p.inboxDir)
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := filepath.Join(p.inboxDir, entry.Name())
|
||||||
|
if err := p.ProcessFile(ctx, path); err != nil {
|
||||||
|
log.Printf("Error processing %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch for new files
|
||||||
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer watcher.Close()
|
||||||
|
|
||||||
|
if err := watcher.Add(p.inboxDir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Watching inbox: %s", p.inboxDir)
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case event, ok := <-watcher.Events:
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if event.Op&fsnotify.Create == fsnotify.Create {
|
||||||
|
// Wait a moment for file to be fully written
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
if err := p.ProcessFile(ctx, event.Name); err != nil {
|
||||||
|
log.Printf("Error processing %s: %v", event.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case err, ok := <-watcher.Errors:
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
log.Printf("Watcher error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) ProcessFile(ctx context.Context, path string) error {
|
||||||
|
// Skip hidden files and non-PDFs/images
|
||||||
|
base := filepath.Base(path)
|
||||||
|
if strings.HasPrefix(base, ".") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := strings.ToLower(filepath.Ext(path))
|
||||||
|
if ext != ".pdf" && ext != ".jpg" && ext != ".jpeg" && ext != ".png" {
|
||||||
|
log.Printf("Skipping non-document file: %s", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Processing: %s", path)
|
||||||
|
|
||||||
|
// Read file
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute checksum
|
||||||
|
hash := sha256.Sum256(data)
|
||||||
|
checksum := hex.EncodeToString(hash[:])
|
||||||
|
|
||||||
|
// Generate ID
|
||||||
|
id := uuid.New().String()
|
||||||
|
|
||||||
|
// Extract text via OCR
|
||||||
|
ocrText, pageCount, err := p.extractText(path, ext)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("OCR failed for %s: %v", path, err)
|
||||||
|
ocrText = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Classify with AI
|
||||||
|
classification, err := p.classify(ctx, ocrText, base)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Classification failed for %s: %v", path, err)
|
||||||
|
classification = &Classification{
|
||||||
|
Category: "uncategorized",
|
||||||
|
Title: base,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store PDF
|
||||||
|
storageName := fmt.Sprintf("%s%s", checksum[:16], ext)
|
||||||
|
storagePath := filepath.Join(p.storeDir, storageName)
|
||||||
|
if err := os.WriteFile(storagePath, data, 0644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse date
|
||||||
|
var docDate *time.Time
|
||||||
|
if classification.Date != "" {
|
||||||
|
if t, err := parseDate(classification.Date); err == nil {
|
||||||
|
docDate = &t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create document record
|
||||||
|
doc := &db.Document{
|
||||||
|
ID: id,
|
||||||
|
Filename: storageName,
|
||||||
|
OriginalName: base,
|
||||||
|
Category: classification.Category,
|
||||||
|
Subcategory: classification.Subcategory,
|
||||||
|
Title: classification.Title,
|
||||||
|
Date: docDate,
|
||||||
|
Vendor: classification.Vendor,
|
||||||
|
Amount: classification.Amount,
|
||||||
|
Currency: classification.Currency,
|
||||||
|
TaxDeductible: classification.TaxDeductible,
|
||||||
|
OCRText: ocrText,
|
||||||
|
StoragePath: storagePath,
|
||||||
|
PageCount: pageCount,
|
||||||
|
FileSize: int64(len(data)),
|
||||||
|
MimeType: getMimeType(ext),
|
||||||
|
Checksum: checksum,
|
||||||
|
ProcessedAt: time.Now(),
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
UpdatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate markdown record
|
||||||
|
mdPath, err := p.writeMarkdown(doc, classification)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to write markdown: %v", err)
|
||||||
|
} else {
|
||||||
|
doc.MarkdownPath = mdPath
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate embedding
|
||||||
|
if embedding, err := p.generateEmbedding(ctx, ocrText); err == nil {
|
||||||
|
doc.Embedding = embedding
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store metadata as JSON
|
||||||
|
if meta, err := json.Marshal(classification.KeyFields); err == nil {
|
||||||
|
doc.Metadata = meta
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert into database
|
||||||
|
if err := p.db.InsertDocument(doc); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove from inbox
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
log.Printf("Failed to remove inbox file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Processed: %s -> %s (%s)", base, classification.Title, classification.Category)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) extractText(path, ext string) (string, int, error) {
|
||||||
|
if ext == ".pdf" {
|
||||||
|
return p.extractPDFText(path)
|
||||||
|
}
|
||||||
|
return p.extractImageText(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) extractPDFText(path string) (string, int, error) {
|
||||||
|
// Try pdftotext first (poppler-utils)
|
||||||
|
cmd := exec.Command("pdftotext", "-layout", path, "-")
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err == nil && len(output) > 100 {
|
||||||
|
// Count pages
|
||||||
|
pageCmd := exec.Command("pdfinfo", path)
|
||||||
|
pageOut, _ := pageCmd.Output()
|
||||||
|
pages := 1
|
||||||
|
if match := regexp.MustCompile(`Pages:\s+(\d+)`).FindSubmatch(pageOut); len(match) > 1 {
|
||||||
|
fmt.Sscanf(string(match[1]), "%d", &pages)
|
||||||
|
}
|
||||||
|
return string(output), pages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to OCR via tesseract
|
||||||
|
// Convert PDF to images first
|
||||||
|
tmpDir, err := os.MkdirTemp("", "docman-ocr-")
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
// Use pdftoppm to convert to images
|
||||||
|
cmd = exec.Command("pdftoppm", "-png", "-r", "300", path, filepath.Join(tmpDir, "page"))
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return "", 0, fmt.Errorf("pdftoppm failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OCR each page
|
||||||
|
var textBuf bytes.Buffer
|
||||||
|
pages, _ := filepath.Glob(filepath.Join(tmpDir, "page-*.png"))
|
||||||
|
for _, pagePath := range pages {
|
||||||
|
text, _, _ := p.extractImageText(pagePath)
|
||||||
|
textBuf.WriteString(text)
|
||||||
|
textBuf.WriteString("\n\n--- Page Break ---\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
return textBuf.String(), len(pages), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) extractImageText(path string) (string, int, error) {
|
||||||
|
cmd := exec.Command("tesseract", path, "stdout", "-l", "eng+nld")
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return "", 1, err
|
||||||
|
}
|
||||||
|
return string(output), 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) classify(ctx context.Context, text, filename string) (*Classification, error) {
|
||||||
|
prompt := fmt.Sprintf(`Analyze this scanned document and extract structured information.
|
||||||
|
|
||||||
|
Document filename: %s
|
||||||
|
|
||||||
|
OCR Text:
|
||||||
|
%s
|
||||||
|
|
||||||
|
Classify and extract the following JSON structure:
|
||||||
|
{
|
||||||
|
"category": "taxes|expenses|bills|medical|contacts|legal|insurance|banking|receipts|correspondence|uncategorized",
|
||||||
|
"subcategory": "more specific category if applicable",
|
||||||
|
"title": "descriptive title for this document",
|
||||||
|
"date": "YYYY-MM-DD if found",
|
||||||
|
"vendor": "company/person name if applicable",
|
||||||
|
"amount": numeric amount if this is a financial document,
|
||||||
|
"currency": "USD" or other currency code,
|
||||||
|
"tax_deductible": true/false if this is a deductible expense,
|
||||||
|
"summary": "one paragraph summary of the document",
|
||||||
|
"key_fields": {"field_name": "value"} for any other important extracted data
|
||||||
|
}
|
||||||
|
|
||||||
|
Categories:
|
||||||
|
- taxes: W-2, 1099, tax returns, deductions
|
||||||
|
- expenses: receipts, invoices for purchases
|
||||||
|
- bills: utility bills, service bills
|
||||||
|
- medical: medical records, prescriptions, EOBs
|
||||||
|
- contacts: business cards, contact info
|
||||||
|
- legal: contracts, agreements, legal documents
|
||||||
|
- insurance: policies, claims
|
||||||
|
- banking: statements, checks
|
||||||
|
- receipts: general purchase receipts
|
||||||
|
- correspondence: letters, emails
|
||||||
|
|
||||||
|
Return ONLY valid JSON.`, filename, truncate(text, 4000))
|
||||||
|
|
||||||
|
resp, err := p.aiClient.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
|
||||||
|
Model: p.aiModel,
|
||||||
|
Messages: []openai.ChatCompletionMessage{
|
||||||
|
{Role: openai.ChatMessageRoleUser, Content: prompt},
|
||||||
|
},
|
||||||
|
Temperature: 0.1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resp.Choices) == 0 {
|
||||||
|
return nil, fmt.Errorf("no response from AI")
|
||||||
|
}
|
||||||
|
|
||||||
|
content := resp.Choices[0].Message.Content
|
||||||
|
// Extract JSON from response
|
||||||
|
content = extractJSON(content)
|
||||||
|
|
||||||
|
var classification Classification
|
||||||
|
if err := json.Unmarshal([]byte(content), &classification); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse classification: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &classification, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) generateEmbedding(ctx context.Context, text string) ([]byte, error) {
|
||||||
|
if p.embedModel == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := p.aiClient.CreateEmbeddings(ctx, openai.EmbeddingRequest{
|
||||||
|
Model: openai.EmbeddingModel(p.embedModel),
|
||||||
|
Input: []string{truncate(text, 8000)},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resp.Data) == 0 {
|
||||||
|
return nil, fmt.Errorf("no embedding returned")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serialize embedding to bytes
|
||||||
|
embData, err := json.Marshal(resp.Data[0].Embedding)
|
||||||
|
return embData, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) writeMarkdown(doc *db.Document, class *Classification) (string, error) {
|
||||||
|
// Create category subdirectory
|
||||||
|
catDir := filepath.Join(p.recordsDir, doc.Category)
|
||||||
|
if err := os.MkdirAll(catDir, 0755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate filename
|
||||||
|
dateStr := "undated"
|
||||||
|
if doc.Date != nil {
|
||||||
|
dateStr = doc.Date.Format("2006-01-02")
|
||||||
|
}
|
||||||
|
safeName := sanitizeFilename(doc.Title)
|
||||||
|
mdName := fmt.Sprintf("%s_%s.md", dateStr, safeName)
|
||||||
|
mdPath := filepath.Join(catDir, mdName)
|
||||||
|
|
||||||
|
// Build markdown content
|
||||||
|
var buf bytes.Buffer
|
||||||
|
buf.WriteString(fmt.Sprintf("# %s\n\n", doc.Title))
|
||||||
|
buf.WriteString("## Metadata\n\n")
|
||||||
|
buf.WriteString(fmt.Sprintf("- **ID:** %s\n", doc.ID))
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Category:** %s", doc.Category))
|
||||||
|
if doc.Subcategory != "" {
|
||||||
|
buf.WriteString(fmt.Sprintf(" / %s", doc.Subcategory))
|
||||||
|
}
|
||||||
|
buf.WriteString("\n")
|
||||||
|
if doc.Date != nil {
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Date:** %s\n", doc.Date.Format("2006-01-02")))
|
||||||
|
}
|
||||||
|
if doc.Vendor != "" {
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Vendor:** %s\n", doc.Vendor))
|
||||||
|
}
|
||||||
|
if doc.Amount != nil {
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Amount:** %s %.2f\n", doc.Currency, *doc.Amount))
|
||||||
|
}
|
||||||
|
if doc.TaxDeductible {
|
||||||
|
buf.WriteString("- **Tax Deductible:** Yes\n")
|
||||||
|
}
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Original File:** %s\n", doc.OriginalName))
|
||||||
|
buf.WriteString(fmt.Sprintf("- **PDF:** [View](%s)\n", doc.StoragePath))
|
||||||
|
buf.WriteString(fmt.Sprintf("- **Processed:** %s\n", doc.ProcessedAt.Format(time.RFC3339)))
|
||||||
|
|
||||||
|
if class.Summary != "" {
|
||||||
|
buf.WriteString("\n## Summary\n\n")
|
||||||
|
buf.WriteString(class.Summary)
|
||||||
|
buf.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(class.KeyFields) > 0 {
|
||||||
|
buf.WriteString("\n## Key Fields\n\n")
|
||||||
|
for k, v := range class.KeyFields {
|
||||||
|
buf.WriteString(fmt.Sprintf("- **%s:** %s\n", k, v))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.WriteString("\n## Full Text (OCR)\n\n```\n")
|
||||||
|
buf.WriteString(doc.OCRText)
|
||||||
|
buf.WriteString("\n```\n")
|
||||||
|
|
||||||
|
if err := os.WriteFile(mdPath, buf.Bytes(), 0644); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return mdPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
|
||||||
|
func truncate(s string, max int) string {
|
||||||
|
if len(s) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:max]
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractJSON(s string) string {
|
||||||
|
// Try to find JSON block
|
||||||
|
start := strings.Index(s, "{")
|
||||||
|
end := strings.LastIndex(s, "}")
|
||||||
|
if start >= 0 && end > start {
|
||||||
|
return s[start : end+1]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDate(s string) (time.Time, error) {
|
||||||
|
formats := []string{
|
||||||
|
"2006-01-02",
|
||||||
|
"01/02/2006",
|
||||||
|
"1/2/2006",
|
||||||
|
"January 2, 2006",
|
||||||
|
"Jan 2, 2006",
|
||||||
|
"2006/01/02",
|
||||||
|
}
|
||||||
|
for _, f := range formats {
|
||||||
|
if t, err := time.Parse(f, s); err == nil {
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Time{}, fmt.Errorf("cannot parse date: %s", s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getMimeType(ext string) string {
|
||||||
|
switch ext {
|
||||||
|
case ".pdf":
|
||||||
|
return "application/pdf"
|
||||||
|
case ".jpg", ".jpeg":
|
||||||
|
return "image/jpeg"
|
||||||
|
case ".png":
|
||||||
|
return "image/png"
|
||||||
|
default:
|
||||||
|
return "application/octet-stream"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeFilename(s string) string {
|
||||||
|
s = strings.ToLower(s)
|
||||||
|
s = regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(s, "-")
|
||||||
|
s = strings.Trim(s, "-")
|
||||||
|
if len(s) > 50 {
|
||||||
|
s = s[:50]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProcessSingle processes a single file and returns the document (for API uploads)
|
||||||
|
func (p *Processor) ProcessSingle(ctx context.Context, data []byte, filename string) (*db.Document, error) {
|
||||||
|
// Write to temp file for processing
|
||||||
|
ext := strings.ToLower(filepath.Ext(filename))
|
||||||
|
tmpFile, err := os.CreateTemp("", "docman-upload-*"+ext)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmpFile.Name())
|
||||||
|
|
||||||
|
if _, err := tmpFile.Write(data); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tmpFile.Close()
|
||||||
|
|
||||||
|
// Use existing process logic but don't delete the temp file
|
||||||
|
// Compute checksum
|
||||||
|
hash := sha256.Sum256(data)
|
||||||
|
checksum := hex.EncodeToString(hash[:])
|
||||||
|
id := uuid.New().String()
|
||||||
|
|
||||||
|
ocrText, pageCount, err := p.extractText(tmpFile.Name(), ext)
|
||||||
|
if err != nil {
|
||||||
|
ocrText = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
classification, err := p.classify(ctx, ocrText, filename)
|
||||||
|
if err != nil {
|
||||||
|
classification = &Classification{
|
||||||
|
Category: "uncategorized",
|
||||||
|
Title: filename,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
storageName := fmt.Sprintf("%s%s", checksum[:16], ext)
|
||||||
|
storagePath := filepath.Join(p.storeDir, storageName)
|
||||||
|
if err := os.WriteFile(storagePath, data, 0644); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var docDate *time.Time
|
||||||
|
if classification.Date != "" {
|
||||||
|
if t, err := parseDate(classification.Date); err == nil {
|
||||||
|
docDate = &t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
doc := &db.Document{
|
||||||
|
ID: id,
|
||||||
|
Filename: storageName,
|
||||||
|
OriginalName: filename,
|
||||||
|
Category: classification.Category,
|
||||||
|
Subcategory: classification.Subcategory,
|
||||||
|
Title: classification.Title,
|
||||||
|
Date: docDate,
|
||||||
|
Vendor: classification.Vendor,
|
||||||
|
Amount: classification.Amount,
|
||||||
|
Currency: classification.Currency,
|
||||||
|
TaxDeductible: classification.TaxDeductible,
|
||||||
|
OCRText: ocrText,
|
||||||
|
StoragePath: storagePath,
|
||||||
|
PageCount: pageCount,
|
||||||
|
FileSize: int64(len(data)),
|
||||||
|
MimeType: getMimeType(ext),
|
||||||
|
Checksum: checksum,
|
||||||
|
ProcessedAt: time.Now(),
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
UpdatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
mdPath, _ := p.writeMarkdown(doc, classification)
|
||||||
|
doc.MarkdownPath = mdPath
|
||||||
|
|
||||||
|
if embedding, err := p.generateEmbedding(ctx, ocrText); err == nil {
|
||||||
|
doc.Embedding = embedding
|
||||||
|
}
|
||||||
|
|
||||||
|
if meta, err := json.Marshal(classification.KeyFields); err == nil {
|
||||||
|
doc.Metadata = meta
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := p.db.InsertDocument(doc); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return doc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchMarkdown searches markdown files directly (fallback when embeddings unavailable)
|
||||||
|
func SearchMarkdown(recordsDir, query string, limit int) ([]*db.SearchResult, error) {
|
||||||
|
var results []*db.SearchResult
|
||||||
|
query = strings.ToLower(query)
|
||||||
|
terms := strings.Fields(query)
|
||||||
|
|
||||||
|
err := filepath.Walk(recordsDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
content := strings.ToLower(string(data))
|
||||||
|
score := 0.0
|
||||||
|
for _, term := range terms {
|
||||||
|
if strings.Contains(content, term) {
|
||||||
|
score += 1.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if score > 0 {
|
||||||
|
// Extract title from first line
|
||||||
|
lines := strings.Split(string(data), "\n")
|
||||||
|
title := strings.TrimPrefix(lines[0], "# ")
|
||||||
|
|
||||||
|
// Find snippet around first match
|
||||||
|
snippet := findSnippet(string(data), terms[0], 100)
|
||||||
|
|
||||||
|
results = append(results, &db.SearchResult{
|
||||||
|
Document: db.Document{
|
||||||
|
Title: title,
|
||||||
|
MarkdownPath: path,
|
||||||
|
},
|
||||||
|
Score: score / float64(len(terms)),
|
||||||
|
Snippet: snippet,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by score descending
|
||||||
|
for i := 0; i < len(results)-1; i++ {
|
||||||
|
for j := i + 1; j < len(results); j++ {
|
||||||
|
if results[j].Score > results[i].Score {
|
||||||
|
results[i], results[j] = results[j], results[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(results) > limit {
|
||||||
|
results = results[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findSnippet(text, term string, radius int) string {
|
||||||
|
lower := strings.ToLower(text)
|
||||||
|
idx := strings.Index(lower, strings.ToLower(term))
|
||||||
|
if idx < 0 {
|
||||||
|
if len(text) > radius*2 {
|
||||||
|
return text[:radius*2] + "..."
|
||||||
|
}
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
start := idx - radius
|
||||||
|
if start < 0 {
|
||||||
|
start = 0
|
||||||
|
}
|
||||||
|
end := idx + len(term) + radius
|
||||||
|
if end > len(text) {
|
||||||
|
end = len(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
snippet := text[start:end]
|
||||||
|
if start > 0 {
|
||||||
|
snippet = "..." + snippet
|
||||||
|
}
|
||||||
|
if end < len(text) {
|
||||||
|
snippet = snippet + "..."
|
||||||
|
}
|
||||||
|
return snippet
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) GetRecordsDir() string {
|
||||||
|
return p.recordsDir
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Processor) GetStoreDir() string {
|
||||||
|
return p.storeDir
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue