diff --git a/.env.example b/.env.example index 842abc6..56209d4 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,6 @@ NEXTAUTH_URL=http://localhost:3000 # Production (change to your domain) # NEXTAUTH_URL=https://your-domain.com -# n8n API (optional - currently using client-side fetch) -N8N_API_URL=https://n8n.example.com/webhook/location - # SMTP Configuration (Fallback when DB config is empty) SMTP_HOST=smtp.gmail.com SMTP_PORT=587 diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..43a381c --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,108 @@ +# Gemini Code Assistant Context + +This document provides a comprehensive overview of the **Location Tracker** project, designed to give the Gemini code assistant the necessary context to understand the codebase, its architecture, and its conventions. + +## 1. Project Overview + +This is a full-stack web application for real-time location tracking, built with Next.js 14. Its primary purpose is to receive location data from [OwnTracks](https://owntracks.org/) mobile clients via an MQTT broker and display the tracks on an interactive map. + +### Key Technologies + +- **Framework**: Next.js 14 (App Router) +- **Language**: TypeScript +- **Database**: Dual SQLite databases using `better-sqlite3`. + - `data/database.sqlite`: For core application data like users, devices, and settings. + - `data/locations.sqlite`: For high-volume, time-series location data. +- **Data Ingestion**: An MQTT subscriber (`lib/mqtt-subscriber.ts`) connects to a Mosquitto broker to receive location updates from OwnTracks clients. An HTTP endpoint (`/api/locations/ingest`) is also available for manual data pushes. +- **Authentication**: `next-auth` (v5) with a credentials (username/password) provider. It supports `ADMIN` and `VIEWER` roles. +- **Frontend**: React, [React Leaflet](https://react-leaflet.js.org/) for mapping, and Tailwind CSS for styling. +- **Deployment**: The project includes a `Dockerfile` and `docker-compose.yml` for containerized deployment alongside a Mosquitto MQTT broker. + +### Core Features + +- Interactive map displaying real-time location data. +- Filtering by device and time range. +- Admin dashboard for user and device management. +- Secure authentication with role-based access control. +- Database maintenance tasks (cleanup, optimization) via API or scripts. + +## 2. Building and Running the Project + +### Prerequisites + +- Node.js 18+ +- npm + +### Development Mode + +1. **Install dependencies:** + ```bash + npm install + ``` + +2. **Initialize the databases:** + This command creates the two SQLite databases in the `data/` directory and seeds them with a default admin user (`admin` / `admin123`). + ```bash + npm run db:init + ``` + +3. **Run the development server:** + This starts the Next.js application on `http://localhost:3000`. It also starts the MQTT subscriber service. + ```bash + npm run dev + ``` + +### Production Mode + +- **Build the application:** + ```bash + npm run build + ``` +- **Start the application:** + ```bash + npm run start + ``` + +### Docker Compose + +The most straightforward way to run the application and its dependent MQTT broker is using Docker Compose. + +1. **Create an environment file:** + Copy `.env.example` to `.env` and fill in the required secrets, especially `AUTH_SECRET`. + +2. **Start the services:** + ```bash + docker-compose up --build + ``` + This will build the Next.js app image and start both the application container and the Mosquitto container. The `data` directory is mounted as a volume to persist the SQLite databases. + +## 3. Architecture and Data Flow + +The application follows a clean, decoupled architecture. + +**Data Ingestion Flow:** +1. An OwnTracks client publishes a location update to a specific MQTT topic (e.g., `owntracks/user/device10`). +2. The Mosquitto broker receives the message. +3. The Next.js application's MQTT Subscriber (`lib/mqtt-subscriber.ts`), which is started on server boot via `instrumentation.ts`, is subscribed to the `owntracks/+/+` topic. +4. Upon receiving a message, the subscriber parses the payload, transforms it into the application's `Location` format, and saves it to `data/locations.sqlite` using the functions in `lib/db.ts`. + +**Data Retrieval Flow:** +1. The user's browser, viewing the map page, periodically fetches data from the `/api/locations` endpoint. +2. This API route queries the `data/locations.sqlite` database, applying any user-specified filters for device or time. +3. The API returns the location data as a JSON response. +4. The frontend `MapView.tsx` component renders the received data as markers and polylines on the Leaflet map. + +**Authentication & Authorization:** +- Users log in via the `/login` page, which uses the NextAuth.js `Credentials` provider. +- The `lib/auth.ts` file contains the NextAuth configuration, validating credentials against the `User` table in `data/database.sqlite`. +- The `middleware.ts` file protects the `/admin` and `/map` routes, checking for a valid session and enforcing role-based access (`ADMIN` vs. `VIEWER`). + +## 4. Development Conventions + +- **Database Access**: All direct database operations are encapsulated within the `lib/db.ts` file. This provides a single, consistent interface (`userDb`, `deviceDb`, `locationDb`) for interacting with the data layer. +- **Authentication**: All authentication and session logic is centralized in `lib/auth.ts`. The `auth()` object exported from this file is the primary entry point for accessing session data on the server. +- **Background Services**: Long-running background services, like the MQTT subscriber, should be initialized in `lib/startup.ts` and triggered from the `instrumentation.ts` file. This is the standard Next.js way to run code on server startup. +- **Configuration**: Environment variables are used for configuration (see `.env.example`). +- **Scripts**: The `scripts/` directory contains Node.js scripts for various administrative tasks, such as database initialization (`init-database.js`), cleanup (`cleanup-old-locations.js`), and testing. They should be run using `node scripts/.js`. +- **Styling**: Utility-first CSS using Tailwind CSS. +- **Types**: TypeScript types are used throughout. Core application types are defined in files like `lib/db.ts` and `types/location.ts`. diff --git a/MQTT_INTEGRATION.md b/MQTT_INTEGRATION.md deleted file mode 100644 index ec52d8c..0000000 --- a/MQTT_INTEGRATION.md +++ /dev/null @@ -1,559 +0,0 @@ -# MQTT Provisioning Integration - -Diese Anleitung beschreibt die Integration des MQTT Provisioning Systems (aus `mosquitto-automation`) in die Location Tracker App. - -## 🎯 Übersicht - -Die Integration vereint zwei vormals separate Systeme: -- **mosquitto-automation**: Device Provisioning und MQTT Credential Management -- **location-tracker-app**: GPS Tracking Visualisierung mit OwnTracks - -### Was wurde integriert? - -✅ **MQTT Credentials Management** - Direkt im Admin Panel -✅ **ACL (Access Control List) Management** - Feine Kontrolle über Topic-Berechtigungen -✅ **Mosquitto Sync** - Password & ACL Files werden automatisch generiert -✅ **MQTT Subscriber** - Direkte Verarbeitung von OwnTracks Messages (kein n8n mehr nötig) -✅ **Docker Compose Setup** - All-in-One Deployment - ---- - -## 📋 Features - -### Admin Panel: MQTT Provisioning - -**Route:** `/admin/mqtt` - -#### Device Provisioning -- Erstelle MQTT Credentials für registrierte Devices -- Automatische Generierung von Username & Passwort -- Passwörter werden mit `mosquitto_passwd` gehasht -- Default ACL Regel: `owntracks/[device-id]/#` (readwrite) - -#### Credentials Management -- Liste aller provisionierten Devices -- Enable/Disable MQTT Zugriff pro Device -- Passwort Regenerierung -- Credentials löschen (inkl. ACL Regeln) - -#### ACL Management -- Custom Topic Patterns definieren -- Berechtigungen: `read`, `write`, `readwrite` -- Wildcard Support mit `#` -- Regeln pro Device verwalten - -#### Mosquitto Sync -- **"Zu Mosquitto Syncen"** Button im Admin Panel -- Generiert `/mosquitto/config/password.txt` -- Generiert `/mosquitto/config/acl.txt` -- Sendet SIGHUP an Mosquitto Container (Config Reload) -- Zeigt ausstehende Änderungen an - ---- - -## 🗄️ Datenbankschema - -### Neue Tabellen - -#### `mqtt_credentials` -```sql -CREATE TABLE mqtt_credentials ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_id TEXT NOT NULL UNIQUE, -- Referenz zu Device Tabelle - mqtt_username TEXT NOT NULL UNIQUE, - mqtt_password_hash TEXT NOT NULL, -- Mosquitto-kompatible Hash - enabled INTEGER DEFAULT 1, -- 0 = disabled, 1 = enabled - created_at TEXT, - updated_at TEXT, - FOREIGN KEY (device_id) REFERENCES Device(id) ON DELETE CASCADE -); -``` - -#### `mqtt_acl_rules` -```sql -CREATE TABLE mqtt_acl_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_id TEXT NOT NULL, - topic_pattern TEXT NOT NULL, -- z.B. "owntracks/device10/#" - permission TEXT NOT NULL, -- read | write | readwrite - created_at TEXT, - FOREIGN KEY (device_id) REFERENCES Device(id) ON DELETE CASCADE -); -``` - -#### `mqtt_sync_status` -```sql -CREATE TABLE mqtt_sync_status ( - id INTEGER PRIMARY KEY CHECK (id = 1), -- Singleton - pending_changes INTEGER DEFAULT 0, - last_sync_at TEXT, - last_sync_status TEXT, -- success | error: ... - created_at TEXT, - updated_at TEXT -); -``` - -### Migration - -```bash -# Datenbanken initialisieren -npm run db:init - -# MQTT Tabellen hinzufügen -node scripts/add-mqtt-tables.js -``` - ---- - -## 🚀 Installation & Setup - -### Voraussetzungen - -- Docker & Docker Compose -- Node.js 20+ (für lokale Entwicklung) - -### 1. Repository Setup - -```bash -cd location-tracker-app - -# Dependencies installieren -npm install - -# .env Datei erstellen -cp .env.example .env -``` - -### 2. Environment Variables - -Bearbeite `.env`: - -```env -# MQTT Configuration -MQTT_BROKER_URL=mqtt://mosquitto:1883 -MQTT_ADMIN_USERNAME=admin -MQTT_ADMIN_PASSWORD=dein-sicheres-passwort - -MOSQUITTO_PASSWORD_FILE=/mosquitto/config/password.txt -MOSQUITTO_ACL_FILE=/mosquitto/config/acl.txt -MOSQUITTO_CONTAINER_NAME=mosquitto - -# NextAuth -NEXTAUTH_URL=http://localhost:3000 -NEXTAUTH_SECRET= - -# Verschlüsselung für SMTP Passwords -ENCRYPTION_KEY= -``` - -### 3. Docker Compose Start - -```bash -# Build und Start -docker-compose up -d - -# Logs verfolgen -docker-compose logs -f - -# Status prüfen -docker-compose ps -``` - -Die App läuft auf: `http://localhost:3000` -Mosquitto MQTT Broker: `mqtt://localhost:1883` - -### 4. Admin Zugang - -**Default Credentials:** -- Username: `admin` -- Password: `admin123` - -⚠️ **Ändere das Passwort nach dem ersten Login!** - ---- - -## 🔧 Entwicklung - -### Lokale Entwicklung (ohne Docker) - -```bash -# 1. Mosquitto extern starten (oder Docker Compose nur Mosquitto) -docker run -d -p 1883:1883 -p 9001:9001 \ - -v $(pwd)/mosquitto.conf:/mosquitto/config/mosquitto.conf \ - -v mosquitto_data:/mosquitto/data \ - eclipse-mosquitto:2 - -# 2. .env anpassen -MQTT_BROKER_URL=mqtt://localhost:1883 - -# 3. Datenbanken initialisieren -npm run db:init -node scripts/add-mqtt-tables.js - -# 4. App starten -npm run dev -``` - -### Neue MQTT Credentials testen - -```bash -# Mit mosquitto_sub testen -mosquitto_sub -h localhost -p 1883 \ - -u "device_10_abc123" \ - -P "dein-generiertes-passwort" \ - -t "owntracks/10/#" \ - -v - -# OwnTracks Message simulieren -mosquitto_pub -h localhost -p 1883 \ - -u "device_10_abc123" \ - -P "dein-generiertes-passwort" \ - -t "owntracks/10/device" \ - -m '{"_type":"location","lat":52.5200,"lon":13.4050,"tst":1234567890,"batt":85,"vel":5.2}' -``` - ---- - -## 📡 MQTT Subscriber - -Der MQTT Subscriber läuft automatisch beim App-Start und verarbeitet OwnTracks Messages. - -### Implementierung - -- **Service:** `lib/mqtt-subscriber.ts` -- **Startup:** `instrumentation.ts` (Next.js Hook) -- **Topic:** `owntracks/+/+` -- **Datenbank:** Schreibt direkt in `locations.sqlite` - -### OwnTracks Message Format - -```json -{ - "_type": "location", - "tid": "XX", - "lat": 52.5200, - "lon": 13.4050, - "tst": 1234567890, - "batt": 85, - "vel": 5.2, - "acc": 10, - "alt": 50 -} -``` - -### Logs - -```bash -# Docker Logs -docker-compose logs -f app - -# Du solltest sehen: -# ✓ Connected to MQTT broker -# ✓ Subscribed to owntracks/+/+ -# ✓ Location saved: device10 at (52.52, 13.405) -``` - ---- - -## 🔐 Sicherheit - -### Mosquitto Authentication - -- **Keine Anonymous Connections:** `allow_anonymous false` -- **Password File:** Mosquitto-kompatible Hashes (SHA512) -- **ACL File:** Topic-basierte Access Control - -### Best Practices - -1. **Starke Admin Passwörter:** Ändere `MQTT_ADMIN_PASSWORD` in `.env` -2. **Device Passwörter:** Auto-generierte Passwörter haben 128 Bit Entropie -3. **ACL Regeln:** Gib Devices nur Zugriff auf ihre eigenen Topics -4. **Docker Socket:** Container benötigt Zugriff für Mosquitto Reload (optional) - -### ACL Beispiele - -```text -# Device darf nur in eigenes Topic schreiben -user device_10_abc123 -topic readwrite owntracks/10/# - -# Device mit zusätzlichem Read-only Topic -user device_11_xyz789 -topic readwrite owntracks/11/# -topic read status/# - -# Admin hat vollen Zugriff -user admin -topic readwrite # -``` - ---- - -## 🐛 Troubleshooting - -### Problem: "Mosquitto configuration reloaded" fehlgeschlagen - -**Symptom:** Nach Sync kommt Warnung "Could not reload Mosquitto automatically" - -**Lösung:** Docker Socket Zugriff fehlt. Entweder: - -```bash -# Option 1: Manuelle Mosquitto Neustart -docker-compose restart mosquitto - -# Option 2: Docker Socket in docker-compose.yml freigeben (bereits konfiguriert) -volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` - -### Problem: MQTT Subscriber verbindet nicht - -**Debug Steps:** - -```bash -# 1. Prüfe Mosquitto läuft -docker-compose ps mosquitto - -# 2. Prüfe Mosquitto Logs -docker-compose logs mosquitto - -# 3. Prüfe App Logs -docker-compose logs app | grep MQTT - -# 4. Teste MQTT Verbindung manuell -mosquitto_sub -h localhost -p 1883 -u admin -P admin -t '#' -``` - -### Problem: Password Hash falsch - -**Symptom:** Authentication failed im Mosquitto Log - -**Lösung:** `mosquitto_passwd` Tool muss im Container verfügbar sein (ist im Dockerfile installiert) - -```bash -# Im Container testen -docker exec -it location-tracker-app mosquitto_passwd -h -``` - -### Problem: ACL Regeln funktionieren nicht - -**Debug:** - -```bash -# ACL File prüfen -docker exec -it location-tracker-app cat /mosquitto/config/acl.txt - -# Mosquitto Logs auf "Access denied" prüfen -docker-compose logs mosquitto | grep -i denied -``` - ---- - -## 📊 API Endpoints - -### MQTT Credentials - -```http -# Liste aller Credentials -GET /api/mqtt/credentials - -# Credential für Device abrufen -GET /api/mqtt/credentials/{device_id} - -# Neue Credentials erstellen -POST /api/mqtt/credentials -Content-Type: application/json - -{ - "device_id": "10", - "auto_generate": true -} - -# Credentials aktualisieren -PATCH /api/mqtt/credentials/{device_id} -Content-Type: application/json - -{ - "regenerate_password": true, - "enabled": true -} - -# Credentials löschen -DELETE /api/mqtt/credentials/{device_id} -``` - -### ACL Rules - -```http -# ACL Regeln für Device -GET /api/mqtt/acl?device_id=10 - -# Neue ACL Regel erstellen -POST /api/mqtt/acl -Content-Type: application/json - -{ - "device_id": "10", - "topic_pattern": "owntracks/10/#", - "permission": "readwrite" -} - -# ACL Regel löschen -DELETE /api/mqtt/acl/{rule_id} -``` - -### Mosquitto Sync - -```http -# Sync Status abrufen -GET /api/mqtt/sync - -# Sync triggern -POST /api/mqtt/sync -``` - -Alle Endpoints erfordern Admin-Authentifizierung (Role: ADMIN). - ---- - -## 🔄 Migration von mosquitto-automation - -Wenn du bereits Devices in `mosquitto-automation` hast: - -### Automatische Migration (TODO) - -```bash -# Script erstellen das aus der alten DB liest -node scripts/migrate-from-mosquitto-automation.js \ - --old-db /pfad/zu/mosquitto-automation/data/devices.db -``` - -### Manuelle Migration - -1. Exportiere Devices aus alter DB: -```sql -SELECT id, name, username, password_hash, permissions -FROM devices -WHERE active = 1; -``` - -2. Erstelle Devices im neuen System über Admin Panel -3. Provisioniere MQTT Credentials manuell -4. Importiere ACL Regeln - ---- - -## 📚 Architektur - -### Komponenten - -``` -┌─────────────────────────────────────────────────────┐ -│ Location Tracker App (Next.js) │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ │ -│ │ Admin Panel │ │ MQTT Service │ │ API │ │ -│ │ /admin/mqtt │ │ Subscriber │ │ Routes │ │ -│ └──────────────┘ └──────────────┘ └──────────┘ │ -│ │ │ │ │ -│ └──────────────────┴────────────────┘ │ -│ │ │ -│ ┌────▼────┐ │ -│ │ SQLite │ │ -│ │ DB │ │ -│ └────┬────┘ │ -│ │ │ -│ ┌────▼────────┐ │ -│ │ Mosquitto │ │ -│ │ Sync Service│ │ -│ └────┬────────┘ │ -└─────────────────────────┼──────────────────────────┘ - │ - ┌───────────▼───────────┐ - │ Mosquitto Broker │ - │ │ - │ • password.txt │ - │ • acl.txt │ - └───────────┬───────────┘ - │ - ┌───────────▼───────────┐ - │ GPS Tracking │ - │ Devices │ - │ (OwnTracks) │ - └───────────────────────┘ -``` - -### Datei-Struktur - -``` -location-tracker-app/ -├── app/ -│ ├── admin/ -│ │ └── mqtt/ -│ │ └── page.tsx # MQTT Provisioning UI -│ └── api/ -│ └── mqtt/ -│ ├── credentials/ # Credentials Management -│ ├── acl/ # ACL Management -│ └── sync/ # Mosquitto Sync -├── lib/ -│ ├── mqtt-db.ts # MQTT Datenbank Operations -│ ├── mqtt-subscriber.ts # MQTT Message Processing -│ ├── mosquitto-sync.ts # Config File Generation -│ └── startup.ts # Service Initialization -├── scripts/ -│ └── add-mqtt-tables.js # Datenbank Migration -├── docker-compose.yml # Docker Setup -├── Dockerfile # App Container -├── mosquitto.conf # Mosquitto Config -└── instrumentation.ts # Next.js Startup Hook -``` - ---- - -## ✨ Vorteile der Integration - -### Vorher (Separate Systeme) - -``` -GPS Device → MQTT → n8n → HTTP API → location-tracker-app → UI - ↓ - mosquitto-automation (separate) -``` - -**Probleme:** -- n8n als zusätzliche Dependency -- Zwei separate Admin Panels -- Keine zentrale User/Device Verwaltung -- Komplexes Setup - -### Nachher (Integriert) - -``` -GPS Device → MQTT → location-tracker-app → UI - ↓ - (integriertes Provisioning) -``` - -**Vorteile:** -✅ Ein Admin Panel für alles -✅ Direkte MQTT Verarbeitung (schneller) -✅ Einfaches Docker Compose Setup -✅ Zentrale Datenbank -✅ Weniger Dependencies - ---- - -## 🎉 Fertig! - -Die Integration ist komplett. Du kannst jetzt: - -1. **Devices verwalten** unter `/admin/devices` -2. **MQTT Credentials provisionieren** unter `/admin/mqtt` -3. **ACL Regeln definieren** im MQTT Panel -4. **Zu Mosquitto syncen** mit einem Klick -5. **GPS Tracking visualisieren** auf der Hauptseite - -Bei Fragen oder Problemen: Siehe Troubleshooting oder check die Logs. - -Happy Tracking! 🚀📍 diff --git a/N8N_INTEGRATION.md b/N8N_INTEGRATION.md deleted file mode 100644 index 1851d67..0000000 --- a/N8N_INTEGRATION.md +++ /dev/null @@ -1,349 +0,0 @@ -# n8n Integration Anleitung - -## Übersicht - -Die poc-app verwendet nun eine **Dual-Datenbank-Architektur** mit lokalem SQLite-Caching: - -- **database.sqlite** - Benutzerkonten, Geräteverwaltung (kritische Daten, wenige Schreibvorgänge) -- **locations.sqlite** - Standortverfolgung Cache (viele Schreibvorgänge, temporär) - -Diese Architektur bietet: -- **Performance**: Schnelle Abfragen aus lokalem SQLite anstatt NocoDB API -- **Skalierbarkeit**: Unbegrenzter Verlauf ohne Paginierungslimits -- **Resilienz**: Auth-System isoliert von der Tracking-Datenbank -- **Flexibilität**: Einfaches Löschen alter Daten - ---- - -## Erforderliche n8n Workflow-Änderungen - -### Aktueller Flow (ALT) - -``` -MQTT Trigger (owntracks/#) - ↓ -MQTT Location verarbeiten (Parse & Transform) - ↓ -Speichere in NocoDB -``` - -### Neuer Flow (ERFORDERLICH) - -``` -MQTT Trigger (owntracks/#) - ↓ -MQTT Location verarbeiten (Parse & Transform) - ↓ -Speichere in NocoDB - ↓ -[NEU] Push to Next.js Cache (HTTP Request) -``` - ---- - -## Schritt-für-Schritt: HTTP Request Node hinzufügen - -### 1. HTTP Request Node hinzufügen - -Nach dem "Speichere in NocoDB" Node einen neuen **HTTP Request** Node hinzufügen: - -**Node-Konfiguration:** -- **Name**: "Push to Next.js Cache" -- **Methode**: POST -- **URL**: `https://deine-nextjs-domain.com/api/locations/ingest` -- **Authentifizierung**: None (API-Key in Produktion hinzufügen!) -- **Body Content Type**: JSON -- **Specify Body**: Using Fields Below - -### 2. Felder zuordnen - -Im Bereich "Body Parameters" die folgenden Felder zuordnen: - -| Parameter | Wert (Expression) | Beschreibung | -|-----------|-------------------|-------------| -| `latitude` | `{{ $json.latitude }}` | Geografischer Breitengrad | -| `longitude` | `{{ $json.longitude }}` | Geografischer Längengrad | -| `timestamp` | `{{ $json.timestamp }}` | ISO 8601 Zeitstempel | -| `user_id` | `{{ $json.user_id }}` | Benutzer-ID (0 für MQTT) | -| `first_name` | `{{ $json.first_name }}` | Tracker ID | -| `last_name` | `{{ $json.last_name }}` | Quelltyp | -| `username` | `{{ $json.username }}` | Geräte-Benutzername | -| `marker_label` | `{{ $json.marker_label }}` | Anzeige-Label | -| `display_time` | `{{ $json.display_time }}` | Formatierte Zeit | -| `chat_id` | `{{ $json.chat_id }}` | Chat-ID (0 für MQTT) | -| `battery` | `{{ $json.battery }}` | Batterieprozent | -| `speed` | `{{ $json.speed }}` | Geschwindigkeit (m/s) | - -### 3. Fehlerbehandlung (Optional aber empfohlen) - -Einen **Error Trigger** Node hinzufügen, um fehlgeschlagene API-Aufrufe zu behandeln: - -- **Workflow**: Current Workflow -- **Error Type**: All Errors -- **Connected to**: Push to Next.js Cache node - -Einen **Slack/Email** Benachrichtigungs-Node hinzufügen, um über Fehler informiert zu werden. - ---- - -## Beispiel n8n HTTP Request Node (JSON) - -```json -{ - "parameters": { - "method": "POST", - "url": "https://deine-domain.com/api/locations/ingest", - "authentication": "none", - "options": {}, - "bodyParametersJson": "={{ {\n \"latitude\": $json.latitude,\n \"longitude\": $json.longitude,\n \"timestamp\": $json.timestamp,\n \"user_id\": $json.user_id,\n \"first_name\": $json.first_name,\n \"last_name\": $json.last_name,\n \"username\": $json.username,\n \"marker_label\": $json.marker_label,\n \"display_time\": $json.display_time,\n \"chat_id\": $json.chat_id,\n \"battery\": $json.battery,\n \"speed\": $json.speed\n} }}" - }, - "name": "Push to Next.js Cache", - "type": "n8n-nodes-base.httpRequest", - "typeVersion": 4.1, - "position": [1200, 300] -} -``` - ---- - -## Testen - -### 1. Ingest-Endpunkt testen - -```bash -curl -X POST https://deine-domain.com/api/locations/ingest \ - -H "Content-Type: application/json" \ - -d '{ - "latitude": 48.1351, - "longitude": 11.5820, - "timestamp": "2024-01-15T10:30:00Z", - "user_id": 0, - "username": "10", - "marker_label": "Test Gerät", - "battery": 85, - "speed": 2.5 - }' -``` - -Erwartete Antwort: -```json -{ - "success": true, - "inserted": 1, - "message": "Successfully stored 1 location(s)" -} -``` - -### 2. Daten überprüfen - -```bash -curl https://deine-domain.com/api/locations?username=10&timeRangeHours=1 -``` - -### 3. Statistiken prüfen - -```bash -curl https://deine-domain.com/api/locations/ingest -``` - ---- - -## Produktions-Überlegungen - -### 1. API-Key-Authentifizierung hinzufügen - -**Aktualisieren** von `app/api/locations/ingest/route.ts`: - -```typescript -export async function POST(request: NextRequest) { - // API-Key validieren - const apiKey = request.headers.get('x-api-key'); - if (apiKey !== process.env.N8N_API_KEY) { - return NextResponse.json( - { error: 'Unauthorized' }, - { status: 401 } - ); - } - - // ... restlicher Code -} -``` - -**n8n HTTP Request Node aktualisieren:** -- Header hinzufügen: `x-api-key` = `dein-secret-key` - -### 2. Automatisches Aufräumen einrichten - -Einen Cron-Job hinzufügen, um alte Daten zu löschen (hält die Datenbankgröße überschaubar): - -```bash -# /etc/crontab -# Tägliche Bereinigung um 2 Uhr morgens - löscht Daten älter als 7 Tage -0 2 * * * cd /pfad/zu/poc-app && node scripts/cleanup-old-locations.js 168 -``` - -Oder verwende einen systemd Timer, PM2 Cron oder ähnliches. - -### 3. Datenbankgröße überwachen - -```bash -# Datenbankstatistiken prüfen -curl https://deine-domain.com/api/locations/ingest - -# Erwartete Ausgabe: -# { -# "total": 5432, -# "oldest": "2024-01-08T10:00:00Z", -# "newest": "2024-01-15T10:30:00Z", -# "sizeKB": 1024 -# } -``` - -### 4. Backup-Strategie - -**database.sqlite** (kritisch): -- Tägliche automatisierte Backups -- 30-Tage Aufbewahrung - -**locations.sqlite** (Cache): -- Optional: Wöchentliche Backups -- Oder keine Backups (Daten existieren in NocoDB) - ---- - -## Migration: Vorhandene NocoDB-Daten importieren - -Falls du bereits Standortdaten in NocoDB hast, kannst du diese importieren: - -### Option 1: CSV aus NocoDB exportieren - -1. Daten aus NocoDB als CSV exportieren -2. In JSON konvertieren -3. In Batches an `/api/locations/ingest` senden (POST) - -### Option 2: Direkter NocoDB API Import (empfohlen) - -Ein Skript `scripts/import-from-nocodb.js` erstellen: - -```javascript -const fetch = require('node-fetch'); -const { locationDb } = require('../lib/db'); - -async function importFromNocoDB() { - // Alle Daten von NocoDB API abrufen - const response = await fetch('https://n8n.example.com/webhook/location'); - const data = await response.json(); - - // Bulk-Insert in locations.sqlite - const count = locationDb.createMany(data.history); - console.log(`Importiert: ${count} Standorte`); -} - -importFromNocoDB().catch(console.error); -``` - -Ausführen: `node scripts/import-from-nocodb.js` - ---- - -## Fehlerbehebung - -### Problem: "directory does not exist" Fehler - -**Lösung**: Init-Skript ausführen: -```bash -cd poc-app -node scripts/init-locations-db.js -``` - -### Problem: n8n gibt 500-Fehler beim Push zu Next.js zurück - -**Prüfen**: -1. Next.js App läuft und ist erreichbar -2. URL in n8n ist korrekt -3. Next.js Logs prüfen: `pm2 logs` oder `docker logs` - -### Problem: Keine Daten im Frontend sichtbar - -**Überprüfen**: -1. Daten sind in SQLite: `curl https://deine-domain.com/api/locations/ingest` -2. API gibt Daten zurück: `curl https://deine-domain.com/api/locations` -3. Browser-Konsole auf Fehler prüfen - -### Problem: Datenbank wird zu groß - -**Lösung**: Cleanup-Skript ausführen: -```bash -node scripts/cleanup-old-locations.js 168 # Behalte 7 Tage -``` - -Oder die Aufbewahrungsdauer im Cron-Job reduzieren. - ---- - -## Architektur-Diagramm - -``` -┌─────────────────┐ -│ OwnTracks App │ -└────────┬────────┘ - │ MQTT - ▼ -┌─────────────────┐ -│ MQTT Broker │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ n8n Workflow │ -│ │ -│ 1. Parse MQTT │ -│ 2. Save NocoDB │───────────┐ -│ 3. Push Next.js│ │ -└────────┬────────┘ │ - │ │ - │ HTTP POST │ (Backup) - ▼ ▼ -┌─────────────────┐ ┌──────────────┐ -│ Next.js API │ │ NocoDB │ -│ /api/locations │ │ (Cloud DB) │ -│ /ingest │ └──────────────┘ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ locations.sqlite│ (Lokaler Cache) -│ - Schnelle │ -│ Abfragen │ -│ - Auto Cleanup │ -│ - Isoliert │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Frontend API │ -│ /api/locations │ -│ (Read-only) │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Map UI │ -│ - 24h Verlauf │ -│ - Schnelle │ -│ Filter │ -│ - Echtzeit │ -└─────────────────┘ -``` - ---- - -## Zusammenfassung - -✅ **Dual-Datenbank** isoliert kritische Auth von hochvolumiger Standortverfolgung -✅ **Lokaler Cache** ermöglicht schnelle 24h+ Abfragen ohne NocoDB-Paginierungslimits -✅ **WAL-Modus** bietet Absturzsicherheit und bessere Nebenläufigkeit -✅ **Auto Cleanup** hält die Datenbankgröße überschaubar -✅ **Rückwärtskompatibel** - gleiches API-Antwortformat wie n8n Webhook - -🚀 **Die Next.js App ist jetzt produktionsreif mit unbegrenztem Standortverlauf!** diff --git a/README.md b/README.md index 4489782..37209ff 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ Eine moderne Location-Tracking Anwendung basierend auf Next.js 14 mit MQTT/OwnTracks Integration, SQLite-Datenbank, Admin-Panel und Authentifizierung. -![Location Tracker Screenshot](./pictures/n8n-MQTT-GPS-Tracking.png) - ## 📋 Inhaltsverzeichnis - [Features](#-features) @@ -43,7 +41,6 @@ Eine moderne Location-Tracking Anwendung basierend auf Next.js 14 mit MQTT/OwnTr - ⏱️ **System Status** - Live-Uptime, Memory Usage, Runtime Info - 📱 **Device Management** - Geräte hinzufügen, bearbeiten, löschen - 💾 **Datenbank-Wartung**: - - 🔄 Manueller Sync von n8n - 🧹 Cleanup alter Daten (7, 15, 30, 90 Tage) - ⚡ Datenbank-Optimierung (VACUUM) - 📈 Detaillierte Statistiken @@ -61,7 +58,7 @@ Eine moderne Location-Tracking Anwendung basierend auf Next.js 14 mit MQTT/OwnTr - **Authentifizierung:** NextAuth.js v5 (beta) - **Datenbank:** SQLite (better-sqlite3) - **Passwort-Hashing:** bcryptjs -- **Datenquelle:** n8n Webhook API + lokale SQLite-Cache +- **Datenquelle:** MQTT Broker + lokale SQLite-Cache ### Dual-Database Architektur - **database.sqlite** - User, Geräte (kritische Daten) @@ -249,9 +246,9 @@ Passwort: admin123 In der OwnTracks App: - **Tracker ID (tid):** z.B. `12` - **Topic:** `owntracks/user/12` -- MQTT Broker wie gewohnt +- MQTT Broker konfigurieren (siehe MQTT Setup) -Die n8n-Workflow holt die Daten, und die App synct automatisch alle 5 Sekunden. +Die App empfängt die Daten direkt vom MQTT Broker. ### Zeitfilter verwenden @@ -292,36 +289,29 @@ Für spezifische Zeiträume, z.B. "Route von gestern Abend 18:00 bis heute Morge ```mermaid flowchart TD A[📱 OwnTracks App] -->|MQTT Publish| B[🔌 MQTT Broker] - B -->|Subscribe| C[⚙️ n8n MQTT Trigger] - C -->|Store| D[💾 NocoDB] - D -->|Webhook API| E[🌐 n8n Webhook
/webhook/location] + B -->|Subscribe| C[📡 Next.js MQTT Subscriber] - F[🖥️ Browser Client] -->|GET /api/locations
alle 5 Sek| G[📡 Next.js API Route] + C -->|Store Locations| D[(🗄️ SQLite Cache
locations.sqlite)] - G -->|1. Fetch Fresh Data| E - E -->|JSON Response| G + E[🖥️ Browser Client] -->|GET /api/locations
alle 5 Sek| F[📡 Next.js API Route] - G -->|2. Sync New Locations| H[(🗄️ SQLite Cache
locations.sqlite)] + D -->|Query Filtered Data| F + F -->|JSON Response| E - H -->|3. Query Filtered Data| G - G -->|JSON Response| F + E -->|Render| G[🗺️ React Leaflet Map] - F -->|Render| I[🗺️ React Leaflet Map] - - J[👤 Admin User] -->|Login| K[🔐 NextAuth.js] - K -->|Authenticated| L[📊 Admin Panel] - L -->|CRUD Operations| M[(💼 SQLite DB
database.sqlite)] + H[👤 Admin User] -->|Login| I[🔐 NextAuth.js] + I -->|Authenticated| J[📊 Admin Panel] + J -->|CRUD Operations| K[(💼 SQLite DB
database.sqlite)] style A fill:#4CAF50 style B fill:#FF9800 style C fill:#2196F3 - style D fill:#9C27B0 - style E fill:#F44336 - style G fill:#00BCD4 - style H fill:#FFC107 - style I fill:#8BC34A - style K fill:#E91E63 - style M fill:#FFC107 + style D fill:#FFC107 + style F fill:#00BCD4 + style G fill:#8BC34A + style I fill:#E91E63 + style K fill:#FFC107 ``` ### Komponenten-Übersicht @@ -331,42 +321,39 @@ graph LR subgraph "External Services" A[OwnTracks App] B[MQTT Broker] - C[n8n Automation] - D[NocoDB] end subgraph "Next.js Application" - E[Frontend
React/Leaflet] - F[API Routes] - G[Auth Layer
NextAuth.js] + C[MQTT Subscriber] + D[Frontend
React/Leaflet] + E[API Routes] + F[Auth Layer
NextAuth.js] end subgraph "Data Layer" - H[locations.sqlite
Tracking Data] - I[database.sqlite
Users & Devices] + G[locations.sqlite
Tracking Data] + H[database.sqlite
Users & Devices] end A -->|MQTT| B B -->|Subscribe| C - C -->|Store| D - C -->|Webhook| F + C -->|Write| G - E -->|HTTP| F - F -->|Read/Write| H - F -->|Read/Write| I + D -->|HTTP| E + E -->|Read/Write| G + E -->|Read/Write| H - E -->|Auth| G - G -->|Validate| I + D -->|Auth| F + F -->|Validate| H style A fill:#4CAF50,color:#fff style B fill:#FF9800,color:#fff style C fill:#2196F3,color:#fff - style D fill:#9C27B0,color:#fff + style D fill:#00BCD4,color:#000 style E fill:#00BCD4,color:#000 - style F fill:#00BCD4,color:#000 - style G fill:#E91E63,color:#fff + style F fill:#E91E63,color:#fff + style G fill:#FFC107,color:#000 style H fill:#FFC107,color:#000 - style I fill:#FFC107,color:#000 ``` ### Datenbank-Architektur @@ -410,20 +397,21 @@ erDiagram } ``` -### Auto-Sync Mechanismus +### Auto-Refresh Mechanismus -Die App verwendet einen **Hybrid-Ansatz**: +Die App verwendet einen **Echtzeit-Ansatz**: -1. **Frontend polling** (alle 5 Sek.) → `/api/locations` -2. **API prüft** ob neue Daten in n8n verfügbar -3. **Nur neue Locations** werden in SQLite gespeichert -4. **Duplikate** werden durch UNIQUE Index verhindert -5. **Antwort** kommt aus lokalem SQLite Cache +1. **MQTT Subscriber** empfängt OwnTracks Messages direkt vom Broker +2. **Locations** werden sofort in SQLite gespeichert +3. **Frontend polling** (alle 5 Sek.) → `/api/locations` +4. **API liest** gefilterte Daten aus lokalem SQLite Cache +5. **Duplikate** werden durch UNIQUE Index verhindert **Vorteile:** -- Schnelle Antwortzeiten (SQLite statt n8n) +- Echtzeitdaten ohne Verzögerung +- Schnelle Antwortzeiten (direkter SQLite Zugriff) - Längere Zeiträume abrufbar (24h+) -- Funktioniert auch wenn n8n nicht erreichbar ist (Fallback auf n8n-Daten) +- Keine externen Dependencies (kein n8n nötig) - Duplikate werden automatisch verhindert (UNIQUE Index) ### Datenvalidierung & Normalisierung @@ -431,18 +419,13 @@ Die App verwendet einen **Hybrid-Ansatz**: Die App behandelt spezielle Fälle bei speed/battery korrekt: **speed = 0 Behandlung:** -- n8n sendet `speed: 0` (gültig - Gerät steht still) +- MQTT sendet `speed: 0` (gültig - Gerät steht still) - Wird mit `typeof === 'number'` Check explizit als `0` gespeichert - Wird NICHT zu `null` konvertiert (wichtig für Telemetrie) - Popup zeigt "Speed: 0.0 km/h" an -**n8n Fallback:** -- Wenn DB leer ist, gibt API direkt n8n-Daten zurück -- Alle Filter (username, timeRangeHours) funktionieren auch mit Fallback -- Ermöglicht sofortigen Betrieb ohne DB-Initialisierung - **Debug-Logging:** -- Server-Logs zeigen n8n Sync-Aktivität +- Server-Logs zeigen MQTT Message-Verarbeitung - Browser Console zeigt Daten-Flow (MapView, Popup) - Hilfreich für Troubleshooting @@ -453,14 +436,13 @@ Die App behandelt spezielle Fälle bei speed/battery korrekt: ### Öffentlich **GET /api/locations** -- Location-Daten abrufen (mit Auto-Sync) +- Location-Daten abrufen (aus SQLite Cache) - Query-Parameter: - `username` - Device-Filter (z.B. "10", "11") - **Zeitfilter (wähle eine Methode):** - `timeRangeHours` - Quick Filter (1, 3, 6, 12, 24) - `startTime` & `endTime` - Custom Range (ISO 8601 Format) - `limit` - Max. Anzahl (Standard: 1000) - - `sync=false` - Nur Cache ohne n8n Sync **Beispiele:** ```bash @@ -493,10 +475,6 @@ GET /api/locations?username=10&startTime=2025-11-16T16:00:00.000Z&endTime=2025-1 **DELETE /api/devices/:id** - Gerät löschen (soft delete) -**POST /api/locations/sync** -- Manueller Sync von n8n -- Gibt Anzahl neu eingefügter Locations zurück - **POST /api/locations/cleanup** - Alte Locations löschen - Body: `{ retentionHours }` @@ -562,14 +540,6 @@ node scripts/optimize-db.js - `VACUUM` - Speicherplatz freigeben - `ANALYZE` - Query-Statistiken aktualisieren -### Sync von n8n - -**Via Admin-Panel:** -- `/admin` → Database Maintenance → Sync Now - -**Automatisch:** -- Passiert alle 5 Sekunden beim Abruf der Karte - ### Logs prüfen ```bash @@ -612,9 +582,6 @@ Erstelle `.env.local`: # NextAuth AUTH_SECRET= NEXTAUTH_URL=https://your-domain.com - -# Optional: n8n API URL (Standard in Code definiert) -N8N_API_URL=https://n8n.example.com/webhook/location ``` **Secret generieren:** @@ -781,9 +748,9 @@ node scripts/init-locations-db.js ### Map zeigt keine Daten -1. n8n Webhook erreichbar? `curl https://n8n.example.com/webhook/location` +1. MQTT Broker erreichbar? `mosquitto_sub -h localhost -p 1883 -t '#'` 2. Locations in Datenbank? `/admin` → Database Statistics prüfen -3. Auto-Sync aktiv? Browser Console öffnen +3. MQTT Subscriber läuft? Server-Logs prüfen ### "ENOENT: no such file or directory, open 'data/database.sqlite'" @@ -861,7 +828,7 @@ Alle vollständigen Lizenztexte der verwendeten Bibliotheken finden Sie in den j - **NextAuth.js** - Authentifizierung - **better-sqlite3** - SQLite für Node.js - **Tailwind CSS** - Utility-First CSS -- **n8n** - Workflow Automation (Backend) +- **MQTT.js** - MQTT Client für Node.js - **OwnTracks** - Location Tracking Apps --- diff --git a/app/api/export/csv/route.ts b/app/api/export/csv/route.ts new file mode 100644 index 0000000..f2a9bb3 --- /dev/null +++ b/app/api/export/csv/route.ts @@ -0,0 +1,143 @@ +import { NextRequest, NextResponse } from "next/server"; +import { locationDb, userDb } from "@/lib/db"; +import { auth } from "@/lib/auth"; +import { calculateDistance, RateLimitedGeocoder } from "@/lib/geo-utils"; + +/** + * GET /api/export/csv + * + * Export location data as CSV for Lexware digital logbook + * + * Query parameters: + * - username: Filter by device tracker ID + * - startTime: Custom range start (ISO string) + * - endTime: Custom range end (ISO string) + * - includeGeocoding: Whether to include addresses (default: false for preview) + */ +export async function GET(request: NextRequest) { + try { + // Check authentication + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Get user's allowed device IDs + const userId = (session.user as any).id; + const role = (session.user as any).role; + const sessionUsername = session.user.name || ''; + + const userDeviceIds = userDb.getAllowedDeviceIds(userId, role, sessionUsername); + + if (userDeviceIds.length === 0) { + return new NextResponse("Keine Geräte verfügbar", { status: 403 }); + } + + const searchParams = request.nextUrl.searchParams; + const username = searchParams.get('username') || undefined; + const startTime = searchParams.get('startTime') || undefined; + const endTime = searchParams.get('endTime') || undefined; + const includeGeocoding = searchParams.get('includeGeocoding') === 'true'; + + // Fetch locations from database + let locations = locationDb.findMany({ + user_id: 0, + username, + startTime, + endTime, + limit: 10000, // High limit for exports + }); + + // Filter by user's allowed devices + locations = locations.filter(loc => loc.username && userDeviceIds.includes(loc.username)); + + if (locations.length === 0) { + return new NextResponse("Keine Daten im gewählten Zeitraum", { status: 404 }); + } + + // Sort chronologically (oldest first) for proper distance calculation + locations.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()); + + // Initialize geocoder if needed + const geocoder = includeGeocoding ? new RateLimitedGeocoder() : null; + + // Build CSV + const csvRows: string[] = []; + + // Header + csvRows.push('Datum,Uhrzeit,Latitude,Longitude,Adresse,Distanz (km),Geschwindigkeit (km/h),Gerät'); + + // Process each location + for (let i = 0; i < locations.length; i++) { + const loc = locations[i]; + const lat = Number(loc.latitude); + const lon = Number(loc.longitude); + + // Calculate distance from previous point + let distance = 0; + if (i > 0) { + const prevLoc = locations[i - 1]; + distance = calculateDistance( + Number(prevLoc.latitude), + Number(prevLoc.longitude), + lat, + lon + ); + } + + // Geocode if requested + let address = `${lat.toFixed(6)}, ${lon.toFixed(6)}`; + if (geocoder) { + address = await geocoder.geocode(lat, lon); + } + + // Format timestamp (German format) + const date = new Date(loc.timestamp); + const dateStr = date.toLocaleDateString('de-DE', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + }); + const timeStr = date.toLocaleTimeString('de-DE', { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + + // Speed (may be null) + const speed = loc.speed != null ? Number(loc.speed).toFixed(1) : ''; + + // Device name + const deviceName = loc.username || 'Unbekannt'; + + // Build CSV row - properly escape address field + const escapedAddress = address.includes(',') ? `"${address}"` : address; + const distanceStr = distance.toFixed(3).replace('.', ','); // German decimal separator + + csvRows.push( + `${dateStr},${timeStr},${lat.toFixed(6)},${lon.toFixed(6)},${escapedAddress},${distanceStr},${speed},${deviceName}` + ); + } + + const csv = csvRows.join('\n'); + + // Return CSV with proper headers + const filename = `fahrtenbuch_${startTime ? new Date(startTime).toISOString().split('T')[0] : 'alle'}_${endTime ? new Date(endTime).toISOString().split('T')[0] : 'alle'}.csv`; + + return new NextResponse(csv, { + headers: { + 'Content-Type': 'text/csv; charset=utf-8', + 'Content-Disposition': `attachment; filename="${filename}"`, + }, + }); + } catch (error) { + console.error("Error exporting CSV:", error); + return NextResponse.json( + { + error: "Failed to export CSV", + details: error instanceof Error ? error.message : "Unknown error" + }, + { status: 500 } + ); + } +} diff --git a/app/api/locations/ingest/route.ts b/app/api/locations/ingest/route.ts index b0d1af0..500167e 100644 --- a/app/api/locations/ingest/route.ts +++ b/app/api/locations/ingest/route.ts @@ -4,8 +4,8 @@ import { locationDb, Location } from '@/lib/db'; /** * POST /api/locations/ingest * - * Endpoint for n8n to push location data to local SQLite cache. - * This is called AFTER n8n stores the data in NocoDB. + * Endpoint for external systems to push location data to local SQLite cache. + * Can be used for bulk imports or external integrations. * * Expected payload (single location or array): * { diff --git a/app/api/locations/route.ts b/app/api/locations/route.ts index 8b5f599..a016d6d 100644 --- a/app/api/locations/route.ts +++ b/app/api/locations/route.ts @@ -3,15 +3,11 @@ import type { LocationResponse } from "@/types/location"; import { locationDb, Location, deviceDb, userDb } from "@/lib/db"; import { auth } from "@/lib/auth"; -const N8N_API_URL = process.env.N8N_API_URL || "https://n8n.example.com/webhook/location"; - /** * GET /api/locations * - * Hybrid approach: - * 1. Fetch fresh data from n8n webhook - * 2. Store new locations in local SQLite cache - * 3. Return filtered data from SQLite (enables 24h+ history) + * Fetches location data from local SQLite cache. + * The MQTT subscriber automatically writes new locations to the cache. * * Query parameters: * - username: Filter by device tracker ID @@ -19,7 +15,6 @@ const N8N_API_URL = process.env.N8N_API_URL || "https://n8n.example.com/webhook/ * - startTime: Custom range start (ISO string) * - endTime: Custom range end (ISO string) * - limit: Maximum number of records (default: 1000) - * - sync: Set to 'false' to skip n8n fetch and read only from cache */ export async function GET(request: NextRequest) { try { @@ -58,87 +53,8 @@ export async function GET(request: NextRequest) { const limit = searchParams.get('limit') ? parseInt(searchParams.get('limit')!, 10) : 1000; - const sync = searchParams.get('sync') !== 'false'; // Default: true - // Variable to store n8n data as fallback - let n8nData: LocationResponse | null = null; - - // Step 1: Optionally fetch and sync from n8n - if (sync) { - try { - const response = await fetch(N8N_API_URL, { - cache: "no-store", - signal: AbortSignal.timeout(3000), // 3 second timeout - }); - - if (response.ok) { - const data: LocationResponse = await response.json(); - - // Debug: Log first location from n8n - if (data.history && data.history.length > 0) { - console.log('[N8N Debug] First location from n8n:', { - username: data.history[0].username, - speed: data.history[0].speed, - speed_type: typeof data.history[0].speed, - speed_exists: 'speed' in data.history[0], - battery: data.history[0].battery, - battery_type: typeof data.history[0].battery, - battery_exists: 'battery' in data.history[0] - }); - } - - // Normalize data: Ensure speed and battery fields exist (treat 0 explicitly) - if (data.history && Array.isArray(data.history)) { - data.history = data.history.map(loc => { - // Generate display_time in German locale (Europe/Berlin timezone) - const displayTime = new Date(loc.timestamp).toLocaleString('de-DE', { - timeZone: 'Europe/Berlin', - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit' - }); - - return { - ...loc, - display_time: displayTime, - // Explicit handling: 0 is valid, only undefined/null → null - speed: typeof loc.speed === 'number' ? loc.speed : (loc.speed !== undefined && loc.speed !== null ? Number(loc.speed) : null), - battery: typeof loc.battery === 'number' ? loc.battery : (loc.battery !== undefined && loc.battery !== null ? Number(loc.battery) : null), - }; - }); - } - - // Store n8n data for fallback - n8nData = data; - - // Store new locations in SQLite - if (data.history && Array.isArray(data.history) && data.history.length > 0) { - // Get latest timestamp from our DB - const stats = locationDb.getStats(); - const lastLocalTimestamp = stats.newest || '1970-01-01T00:00:00Z'; - - // Filter for only newer locations - const newLocations = data.history.filter(loc => - loc.timestamp > lastLocalTimestamp - ); - - if (newLocations.length > 0) { - const inserted = locationDb.createMany(newLocations as Location[]); - console.log(`[Location Sync] Inserted ${inserted} new locations from n8n`); - } - } - } - } catch (syncError) { - // n8n not reachable - that's ok, we'll use cached data - console.warn('[Location Sync] n8n webhook not reachable, using cache only:', - syncError instanceof Error ? syncError.message : 'Unknown error'); - } - } - - // Step 2: Read from local SQLite with filters + // Read from local SQLite with filters let locations = locationDb.findMany({ user_id: 0, // Always filter for MQTT devices username, @@ -151,38 +67,6 @@ export async function GET(request: NextRequest) { // Filter locations to only include user's devices locations = locations.filter(loc => loc.username && userDeviceIds.includes(loc.username)); - // Step 3: If DB is empty, use n8n data as fallback - if (locations.length === 0 && n8nData && n8nData.history) { - console.log('[API] DB empty, using n8n data as fallback'); - // Filter n8n data if needed - let filteredHistory = n8nData.history; - - // Filter by user's devices - filteredHistory = filteredHistory.filter(loc => loc.username && userDeviceIds.includes(loc.username)); - - if (username) { - filteredHistory = filteredHistory.filter(loc => loc.username === username); - } - - // Apply time filters - if (startTime && endTime) { - // Custom range - filteredHistory = filteredHistory.filter(loc => - loc.timestamp >= startTime && loc.timestamp <= endTime - ); - } else if (timeRangeHours) { - // Quick filter - const cutoffTime = new Date(Date.now() - timeRangeHours * 60 * 60 * 1000).toISOString(); - filteredHistory = filteredHistory.filter(loc => loc.timestamp >= cutoffTime); - } - - return NextResponse.json({ - ...n8nData, - history: filteredHistory, - total_points: filteredHistory.length, - }); - } - // Normalize locations: Ensure speed, battery, and display_time are correct locations = locations.map(loc => { // Generate display_time if missing or regenerate from timestamp @@ -207,7 +91,7 @@ export async function GET(request: NextRequest) { // Get actual total count from database (not limited by 'limit' parameter) const stats = locationDb.getStats(); - // Step 4: Return data in n8n-compatible format + // Return data in standard format const response: LocationResponse = { success: true, current: locations.length > 0 ? locations[0] : null, diff --git a/app/api/locations/sync/route.ts b/app/api/locations/sync/route.ts deleted file mode 100644 index 113d732..0000000 --- a/app/api/locations/sync/route.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { NextResponse } from 'next/server'; -import { auth } from '@/lib/auth'; -import { locationDb, Location } from '@/lib/db'; -import type { LocationResponse } from "@/types/location"; - -const N8N_API_URL = process.env.N8N_API_URL || "https://n8n.example.com/webhook/location"; - -/** - * POST /api/locations/sync (ADMIN only) - * - * Manually sync location data from n8n webhook to local SQLite cache. - * This fetches all available data from n8n and stores only new records. - * - * Useful for: - * - Initial database population - * - Recovery after downtime - * - Manual refresh - */ -export async function POST() { - try { - // ADMIN only - const session = await auth(); - if (!session?.user || (session.user as any).role !== 'ADMIN') { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } - // Get stats before sync - const statsBefore = locationDb.getStats(); - - // Fetch from n8n webhook - const response = await fetch(N8N_API_URL, { - cache: "no-store", - signal: AbortSignal.timeout(10000), // 10 second timeout for manual sync - }); - - if (!response.ok) { - throw new Error(`n8n webhook returned ${response.status}`); - } - - const data: LocationResponse = await response.json(); - - let insertedCount = 0; - - // Store new locations in SQLite - if (data.history && Array.isArray(data.history) && data.history.length > 0) { - // Get latest timestamp from our DB - const lastLocalTimestamp = statsBefore.newest || '1970-01-01T00:00:00Z'; - - // Filter for only newer locations - const newLocations = data.history.filter(loc => - loc.timestamp > lastLocalTimestamp - ); - - if (newLocations.length > 0) { - insertedCount = locationDb.createMany(newLocations as Location[]); - console.log(`[Manual Sync] Inserted ${insertedCount} new locations from n8n`); - } - } - - // Get stats after sync - const statsAfter = locationDb.getStats(); - - return NextResponse.json({ - success: true, - synced: insertedCount, - n8nTotal: data.total_points || data.history.length, - before: { - total: statsBefore.total, - newest: statsBefore.newest, - }, - after: { - total: statsAfter.total, - newest: statsAfter.newest, - }, - }); - - } catch (error) { - console.error('Sync error:', error); - return NextResponse.json( - { - error: 'Failed to sync locations', - details: error instanceof Error ? error.message : 'Unknown error' - }, - { status: 500 } - ); - } -} diff --git a/app/export/page.tsx b/app/export/page.tsx new file mode 100644 index 0000000..2e203ab --- /dev/null +++ b/app/export/page.tsx @@ -0,0 +1,416 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { useSession } from "next-auth/react"; +import { useRouter } from "next/navigation"; + +interface Device { + id: string; + name: string; + color: string; +} + +interface PreviewRow { + datum: string; + uhrzeit: string; + latitude: string; + longitude: string; + adresse: string; + distanz: string; + geschwindigkeit: string; + geraet: string; +} + +export default function ExportPage() { + const { data: session, status } = useSession(); + const router = useRouter(); + + const [devices, setDevices] = useState([]); + const [selectedDevice, setSelectedDevice] = useState("all"); + const [startTime, setStartTime] = useState(""); + const [endTime, setEndTime] = useState(""); + const [previewData, setPreviewData] = useState([]); + const [totalPoints, setTotalPoints] = useState(0); + const [loading, setLoading] = useState(false); + const [exporting, setExporting] = useState(false); + const [exportProgress, setExportProgress] = useState(""); + + // Redirect if not authenticated + useEffect(() => { + if (status === "unauthenticated") { + router.push("/login"); + } + }, [status, router]); + + // Fetch devices + useEffect(() => { + const fetchDevices = async () => { + try { + const response = await fetch("/api/devices/public"); + if (response.ok) { + const data = await response.json(); + setDevices(data.devices || []); + } + } catch (err) { + console.error("Failed to fetch devices:", err); + } + }; + + if (status === "authenticated") { + fetchDevices(); + } + }, [status]); + + // Set default time range (last 7 days) + useEffect(() => { + const now = new Date(); + const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); + + // Format for datetime-local input (YYYY-MM-DDTHH:mm) + setEndTime(now.toISOString().slice(0, 16)); + setStartTime(sevenDaysAgo.toISOString().slice(0, 16)); + }, []); + + // Generate preview + const handlePreview = async () => { + setLoading(true); + setPreviewData([]); + setTotalPoints(0); + + try { + const params = new URLSearchParams(); + if (selectedDevice !== "all") { + params.set("username", selectedDevice); + } + if (startTime) { + params.set("startTime", new Date(startTime).toISOString()); + } + if (endTime) { + params.set("endTime", new Date(endTime).toISOString()); + } + params.set("limit", "50"); // Preview only first 50 + + const response = await fetch(`/api/locations?${params.toString()}`); + if (!response.ok) throw new Error("Failed to fetch preview"); + + const data = await response.json(); + const locations = data.history || []; + setTotalPoints(locations.length); + + // Sort chronologically + locations.sort((a: any, b: any) => + new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime() + ); + + // Build preview rows (without geocoding) + const rows: PreviewRow[] = []; + for (let i = 0; i < Math.min(locations.length, 50); i++) { + const loc = locations[i]; + const lat = Number(loc.latitude); + const lon = Number(loc.longitude); + + // Calculate distance + let distance = 0; + if (i > 0) { + const prevLoc = locations[i - 1]; + distance = calculateDistance( + Number(prevLoc.latitude), + Number(prevLoc.longitude), + lat, + lon + ); + } + + const date = new Date(loc.timestamp); + rows.push({ + datum: date.toLocaleDateString('de-DE'), + uhrzeit: date.toLocaleTimeString('de-DE'), + latitude: lat.toFixed(6), + longitude: lon.toFixed(6), + adresse: `${lat.toFixed(6)}, ${lon.toFixed(6)}`, + distanz: distance.toFixed(3), + geschwindigkeit: loc.speed != null ? Number(loc.speed).toFixed(1) : '-', + geraet: loc.username || 'Unbekannt', + }); + } + + setPreviewData(rows); + } catch (error) { + console.error("Preview error:", error); + alert("Fehler beim Laden der Vorschau"); + } finally { + setLoading(false); + } + }; + + // Export CSV with geocoding + const handleExport = async () => { + if (previewData.length === 0) { + alert("Bitte zuerst Vorschau laden"); + return; + } + + const confirmed = confirm( + `${totalPoints} GPS-Punkte werden exportiert.\n\n` + + `ACHTUNG: Adressauflösung (Geocoding) kann bei vielen Punkten sehr lange dauern (ca. 1 Punkt pro Sekunde).\n\n` + + `Geschätzte Dauer: ca. ${Math.ceil(totalPoints / 60)} Minuten.\n\n` + + `Möchten Sie fortfahren?` + ); + + if (!confirmed) return; + + setExporting(true); + setExportProgress(`Starte Export von ${totalPoints} Punkten...`); + + try { + const params = new URLSearchParams(); + if (selectedDevice !== "all") { + params.set("username", selectedDevice); + } + if (startTime) { + params.set("startTime", new Date(startTime).toISOString()); + } + if (endTime) { + params.set("endTime", new Date(endTime).toISOString()); + } + params.set("includeGeocoding", "true"); + + setExportProgress("Lade Daten und löse Adressen auf..."); + + const response = await fetch(`/api/export/csv?${params.toString()}`); + if (!response.ok) throw new Error("Export failed"); + + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `fahrtenbuch_${startTime ? new Date(startTime).toISOString().split('T')[0] : 'alle'}_${endTime ? new Date(endTime).toISOString().split('T')[0] : 'alle'}.csv`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + setExportProgress("Export erfolgreich abgeschlossen!"); + setTimeout(() => setExportProgress(""), 3000); + } catch (error) { + console.error("Export error:", error); + alert("Fehler beim Exportieren"); + setExportProgress(""); + } finally { + setExporting(false); + } + }; + + // Export CSV without geocoding (faster) + const handleQuickExport = async () => { + if (previewData.length === 0) { + alert("Bitte zuerst Vorschau laden"); + return; + } + + setExporting(true); + setExportProgress("Exportiere ohne Adressauflösung..."); + + try { + const params = new URLSearchParams(); + if (selectedDevice !== "all") { + params.set("username", selectedDevice); + } + if (startTime) { + params.set("startTime", new Date(startTime).toISOString()); + } + if (endTime) { + params.set("endTime", new Date(endTime).toISOString()); + } + params.set("includeGeocoding", "false"); + + const response = await fetch(`/api/export/csv?${params.toString()}`); + if (!response.ok) throw new Error("Export failed"); + + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `fahrtenbuch_schnell_${startTime ? new Date(startTime).toISOString().split('T')[0] : 'alle'}_${endTime ? new Date(endTime).toISOString().split('T')[0] : 'alle'}.csv`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + setExportProgress("Export erfolgreich abgeschlossen!"); + setTimeout(() => setExportProgress(""), 3000); + } catch (error) { + console.error("Export error:", error); + alert("Fehler beim Exportieren"); + setExportProgress(""); + } finally { + setExporting(false); + } + }; + + if (status === "loading") { + return ( +
+

Laden...

+
+ ); + } + + return ( +
+
+
+

CSV-Export für Fahrtenbuch

+

+ Exportieren Sie Ihre GPS-Tracking-Daten für Lexware oder andere Fahrtenbuch-Software +

+ + {/* Filters */} +
+
+ + +
+ +
+ + setStartTime(e.target.value)} + className="w-full border border-gray-300 rounded px-3 py-2" + /> +
+ +
+ + setEndTime(e.target.value)} + className="w-full border border-gray-300 rounded px-3 py-2" + /> +
+
+ + {/* Action Buttons */} +
+ + + + + +
+ + {/* Progress */} + {exportProgress && ( +
+

{exportProgress}

+
+ )} + + {/* Info */} + {totalPoints > 0 && ( +
+

+ Gefunden: {totalPoints} GPS-Punkte im gewählten Zeitraum + {totalPoints > 50 && " (Vorschau zeigt erste 50)"} +

+

+ Hinweis: Der vollständige Export mit Adressauflösung dauert ca. {Math.ceil(totalPoints / 60)} Minuten. + Für schnelle Exporte nutzen Sie den Schnell-Export. +

+
+ )} +
+ + {/* Preview Table */} + {previewData.length > 0 && ( +
+

Vorschau (erste 50 Zeilen)

+
+ + + + + + + + + + + + + + + {previewData.map((row, idx) => ( + + + + + + + + + + + ))} + +
DatumUhrzeitLatitudeLongitudeAdresseDistanz (km)Geschw. (km/h)Gerät
{row.datum}{row.uhrzeit}{row.latitude}{row.longitude}{row.adresse}{row.distanz}{row.geschwindigkeit}{row.geraet}
+
+
+ )} +
+
+ ); +} + +// Haversine distance calculation (client-side for preview) +function calculateDistance(lat1: number, lon1: number, lat2: number, lon2: number): number { + const R = 6371; + const dLat = toRadians(lat2 - lat1); + const dLon = toRadians(lon2 - lon1); + + const a = + Math.sin(dLat / 2) * Math.sin(dLat / 2) + + Math.cos(toRadians(lat1)) * + Math.cos(toRadians(lat2)) * + Math.sin(dLon / 2) * + Math.sin(dLon / 2); + + const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + return R * c; +} + +function toRadians(degrees: number): number { + return degrees * (Math.PI / 180); +} diff --git a/app/map/page.tsx b/app/map/page.tsx index c000b96..3de0f29 100644 --- a/app/map/page.tsx +++ b/app/map/page.tsx @@ -69,12 +69,20 @@ export default function MapPage() { {/* Top row: Title and Admin link */} {/* Controls row - responsive grid */} diff --git a/components/map/MapView.tsx b/components/map/MapView.tsx index 03366a6..d0bfc7a 100644 --- a/components/map/MapView.tsx +++ b/components/map/MapView.tsx @@ -157,7 +157,6 @@ export default function MapView({ selectedDevice, timeFilter, isPaused, filterMo } params.set("limit", "5000"); // Fetch more data for better history - params.set("sync", "false"); // Disable n8n sync (using direct MQTT) // Fetch from local SQLite API (MQTT subscriber writes directly) const response = await fetch(`/api/locations?${params.toString()}`); diff --git a/lib/geo-utils.ts b/lib/geo-utils.ts new file mode 100644 index 0000000..84a693e --- /dev/null +++ b/lib/geo-utils.ts @@ -0,0 +1,111 @@ +/** + * Geo utilities for distance calculation and geocoding + */ + +/** + * Calculate distance between two GPS coordinates using Haversine formula + * @param lat1 Latitude of first point + * @param lon1 Longitude of first point + * @param lat2 Latitude of second point + * @param lon2 Longitude of second point + * @returns Distance in kilometers + */ +export function calculateDistance( + lat1: number, + lon1: number, + lat2: number, + lon2: number +): number { + const R = 6371; // Earth's radius in kilometers + const dLat = toRadians(lat2 - lat1); + const dLon = toRadians(lon2 - lon1); + + const a = + Math.sin(dLat / 2) * Math.sin(dLat / 2) + + Math.cos(toRadians(lat1)) * + Math.cos(toRadians(lat2)) * + Math.sin(dLon / 2) * + Math.sin(dLon / 2); + + const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + const distance = R * c; + + return distance; +} + +function toRadians(degrees: number): number { + return degrees * (Math.PI / 180); +} + +/** + * Reverse geocode coordinates to address using Nominatim (OpenStreetMap) + * @param lat Latitude + * @param lon Longitude + * @returns Address string or coordinates if geocoding fails + */ +export async function reverseGeocode( + lat: number, + lon: number +): Promise { + try { + const url = `https://nominatim.openstreetmap.org/reverse?format=json&lat=${lat}&lon=${lon}&zoom=18&addressdetails=1`; + + const response = await fetch(url, { + headers: { + 'User-Agent': 'GPS-Tracker-App/1.0', // Nominatim requires User-Agent + }, + }); + + if (!response.ok) { + throw new Error(`Nominatim API error: ${response.status}`); + } + + const data = await response.json(); + + if (data.error) { + return `${lat.toFixed(6)}, ${lon.toFixed(6)}`; + } + + // Build address from components + const address = data.address; + const parts: string[] = []; + + if (address.road) parts.push(address.road); + if (address.house_number) parts.push(address.house_number); + if (address.postcode) parts.push(address.postcode); + if (address.city || address.town || address.village) { + parts.push(address.city || address.town || address.village); + } + + return parts.length > 0 ? parts.join(', ') : data.display_name; + } catch (error) { + console.error('Geocoding error:', error); + // Fallback to coordinates + return `${lat.toFixed(6)}, ${lon.toFixed(6)}`; + } +} + +/** + * Rate-limited reverse geocoding (max 1 request per second for Nominatim) + */ +export class RateLimitedGeocoder { + private lastRequestTime = 0; + private minInterval = 1000; // 1 second between requests + + async geocode(lat: number, lon: number): Promise { + const now = Date.now(); + const timeSinceLastRequest = now - this.lastRequestTime; + + if (timeSinceLastRequest < this.minInterval) { + const waitTime = this.minInterval - timeSinceLastRequest; + await this.sleep(waitTime); + } + + this.lastRequestTime = Date.now(); + return reverseGeocode(lat, lon); + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/pictures/n8n-MQTT-GPS-Tracking.png b/pictures/n8n-MQTT-GPS-Tracking.png deleted file mode 100644 index a0394cc..0000000 Binary files a/pictures/n8n-MQTT-GPS-Tracking.png and /dev/null differ