From bcdcbc4a785bd5751d768e468184bb898d314a4f Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 12 Mar 2026 06:46:29 +0800 Subject: [PATCH] docs(phase-02): create phase plans for file management --- .../02-01-PLAN.md | 181 ++++++++++++++++ .../02-02-PLAN.md | 195 ++++++++++++++++++ .../02-RESEARCH.md | 163 +++++++++++++++ .../02-VALIDATION.md | 89 ++++++++ 4 files changed, 628 insertions(+) create mode 100644 .planning/phases/02-data-persistence-file-management/02-01-PLAN.md create mode 100644 .planning/phases/02-data-persistence-file-management/02-02-PLAN.md create mode 100644 .planning/phases/02-data-persistence-file-management/02-RESEARCH.md create mode 100644 .planning/phases/02-data-persistence-file-management/02-VALIDATION.md diff --git a/.planning/phases/02-data-persistence-file-management/02-01-PLAN.md b/.planning/phases/02-data-persistence-file-management/02-01-PLAN.md new file mode 100644 index 000000000..07c0c8c53 --- /dev/null +++ b/.planning/phases/02-data-persistence-file-management/02-01-PLAN.md @@ -0,0 +1,181 @@ +--- +phase: 02-data-persistence-file-management +plan: "01" +type: execute +wave: 1 +depends_on: [] +files_modified: + - src/app.py + - src/templates/files.html +autonomous: true +requirements: + - FILE-01 + - FILE-02 + - FILE-04 + - BACK-04 + +must_haves: + truths: + - "User can view list of CSV files stored in flash memory" + - "User can view contents of selected CSV files" + - "User can delete unwanted CSV files from flash memory" + artifacts: + - path: "src/app.py" + provides: "File management API endpoints" + exports: ["GET /api/files", "GET /api/files/", "DELETE /api/files/"] + - path: "src/templates/files.html" + provides: "File management UI page" + routes: ["/files"] + key_links: + - from: "src/templates/files.html" + to: "/api/files" + via: "fetch in useEffect" + pattern: "fetch.*api/files" +--- + + +Create file management API endpoints and basic UI for listing/viewing/deleting CSV files. + + + +@./.planning/phases/02-data-persistence-file-management/02-CONTEXT.md +@./.planning/phases/02-data-persistence-file-management/02-RESEARCH.md +@./.planning/ROADMAP.md +@./.planning/REQUIREMENTS.md +@./src/app.py + + + + + + task 1: Add file management API endpoints to Flask backend + src/app.py + +Add the following API endpoints to src/app.py: + +1. GET /api/files - List all CSV files in the logger directory + - Returns JSON: {files: [{name, size, created}], total: count} + - Storage path: /myvscada/logger (configurable, default to src/data/logger) + - Handle missing directory gracefully + +2. GET /api/files/<filename> - Get file contents + - Returns JSON: {name, content: "full file text", lines: count} + - Limit preview to first 100 lines for large files + +3. DELETE /api/files/<filename> - Delete a file + - Returns JSON: {success: true/false} + - Confirm file exists before delete + - Handle errors gracefully + +Use the existing Flask patterns from src/app.py: +- Use jsonify for responses +- Use request.get_json() for POST data +- Add error handling with try/except +- Follow existing route structure + + +curl -s http://localhost:8080/api/files | python3 -c "import sys,json; d=json.load(sys.stdin); print('OK' if 'files' in d else 'FAIL')" + + API endpoints return valid JSON, file list shows files, delete removes files + + + + task 2: Create files.html template + src/templates/files.html + +Create src/templates/files.html with: + +1. Main navigation menu (reuse from other templates) +2. Page title: "Flash Memory Files" +3. File list display: + - Table or card layout showing: filename, size, date + - "View" button for each file (opens modal) + - "Delete" button with confirmation +4. Modal for viewing file contents: + - File name header + - Scrollable content area (pre/code block) + - Close button +5. JavaScript: + - Fetch file list on page load + - Handle view button click -> fetch file content + - Handle delete button click -> confirm -> delete -> refresh list + - Use Bootstrap 5.3 + Bootstrap Icons (like existing templates) + - Touch-optimized 48px+ buttons + +Base on the existing templates (settings.html, calibration.html) for: +- Bootstrap 5.3 structure +- Navigation menu +- CSS variables and touch targets +- Socket.IO integration (optional) + + +curl -s http://localhost:8080/files | grep -q "Flash Memory" && echo "OK" || echo "FAIL" + + Files page loads, shows file list, view modal works, delete works + + + + task 3: Add /files route to Flask app + src/app.py + +Add the route for the files page in src/app.py: + +```python +@app.route('/files') +def files(): + """File management page""" + return render_template('files.html', + page='files', + version=SOFTWARE_VERSION, + build_date=BUILD_DATE) +``` + +Add this route near the other page routes (index, settings, calibration). + + +curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/files + + Route returns 200 OK + + + + task 4: Add Files menu item to navigation + src/templates/dashboard.html, src/templates/settings.html, src/templates/calibration.html + +Add "Files" navigation button to main menu in: +- dashboard.html +- settings.html +- calibration.html + +The navigation should link to /files route. Follow the existing nav structure with nav-btn class. + +Example (add after calibration button): +```html + + Files + +``` + + +grep -l "Files" src/templates/*.html | wc -l + + All pages have Files link in navigation + + + + + +- API endpoints return valid JSON +- Files page loads with file list +- View modal shows file contents +- Delete removes files from storage +- Navigation works across all pages + + + +User can view list of CSV files, view file contents, and delete files from flash memory via web UI + + + +After completion, create `.planning/phases/02-data-persistence-file-management/02-01-SUMMARY.md` + diff --git a/.planning/phases/02-data-persistence-file-management/02-02-PLAN.md b/.planning/phases/02-data-persistence-file-management/02-02-PLAN.md new file mode 100644 index 000000000..633653933 --- /dev/null +++ b/.planning/phases/02-data-persistence-file-management/02-02-PLAN.md @@ -0,0 +1,195 @@ +--- +phase: 02-data-persistence-file-management +plan: "02" +type: execute +wave: 1 +depends_on: [] +files_modified: + - src/app.py + - src/templates/files.html +autonomous: true +requirements: + - FILE-03 + - FILE-05 + +must_haves: + truths: + - "User can navigate file list (scroll up/down) to find specific files" + - "User can generate tidEDA formatted files for data transmission" + artifacts: + - path: "src/app.py" + provides: "tidEDA export API endpoint" + exports: ["POST /api/files/export"] + - path: "src/templates/files.html" + provides: "Scroll navigation and export functionality" + key_links: + - from: "src/templates/files.html" + to: "/api/files/export" + via: "fetch POST" + pattern: "fetch.*api/files/export" +--- + + +Add scroll navigation to file list and tidEDA export functionality. + + + +@./.planning/phases/02-data-persistence-file-management/02-CONTEXT.md +@./.planning/phases/02-data-persistence-file-management/02-RESEARCH.md +@./.planning/phases/02-data-persistence-file-management/02-01-PLAN.md +@./src/app.py + + + + + + task 1: Add scroll navigation to file list + src/templates/files.html + +Enhance src/templates/files.html with scroll navigation: + +1. If file list exceeds viewport: + - Enable native scroll (overflow-y: auto) + - Or add pagination controls (Previous/Next buttons) + +2. For touch-optimized scrolling on 7" display: + - Ensure smooth scrolling with -webkit-overflow-scrolling: touch + - Minimum touch target size: 48px for all interactive elements + +3. Add file count display: + - "Showing X of Y files" + - Update after scroll/filter + +4. Optional enhancements: + - Sort by date (newest first) + - Filter by filename search + - These are "OpenCode's Discretion" - implement if straightforward + +The key requirement is FILE-03: User can navigate file list (scroll up/down) to find specific files. + + +Test by loading files page with multiple files and verifying scroll works + + User can scroll through file list to find specific files + + + + task 2: Add tidEDA export API endpoint + src/app.py + +Add POST /api/files/export endpoint to src/app.py: + +```python +@app.route('/api/files/export', methods=['POST']) +def api_files_export(): + """Generate tidEDA formatted export from CSV file""" + data = request.get_json() or {} + filename = data.get('filename') + + if not filename: + return jsonify({"error": "filename required"}), 400 + + # Validate filename (security: prevent path traversal) + if '..' in filename or '/' in filename or '\\' in filename: + return jsonify({"error": "invalid filename"}), 400 + + # Ensure .csv extension + if not filename.endswith('.csv'): + filename = filename + '.csv' + + # Read the CSV file + file_path = os.path.join(LOGGER_DIR, filename) + if not os.path.exists(file_path): + return jsonify({"error": "file not found"}), 404 + + # Generate tidEDA format + # Structure: + # $STATION,{station_id} + # $DATETIME,{timestamp} + # $DATA + # {original_csv_content} + # $END + + settings = load_settings() + station_id = settings.get("station", {}).get("id", "UNKNOWN") + + with open(file_path, 'r') as f: + csv_content = f.read() + + tideda_content = f"$STATION,{station_id}\n" + tideda_content += f"$DATETIME,{datetime.now().isoformat()}\n" + tideda_content += "$DATA\n" + tideda_content += csv_content + tideda_content += "$END\n" + + return jsonify({ + "filename": filename.replace('.csv', '.tideda'), + "content": tideda_content, + "format": "tidEDA v1.0" + }) +``` + +Add LOGGER_DIR constant at top of file: +```python +LOGGER_DIR = os.path.join(os.path.dirname(__file__), 'data', 'logger') +os.makedirs(LOGGER_DIR, exist_ok=True) +``` + + +curl -s -X POST -H "Content-Type: application/json" -d '{"filename":"test.csv"}' http://localhost:8080/api/files/export + + API returns tidEDA formatted content with $STATION, $DATETIME, $DATA, $END structure + + + + task 3: Add export button to files UI + src/templates/files.html + +Enhance src/templates/files.html with export functionality: + +1. Add "Export" button next to each file or as a bulk action: + - Button with download icon + - Calls POST /api/files/export + +2. Handle export response: + - Receive tidEDA formatted content + - Trigger browser download as .tideda file + +3. Add JavaScript download helper: +```javascript +function downloadFile(filename, content) { + const blob = new Blob([content], {type: 'text/plain'}); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); +} +``` + +4. Add "Generate tidEDA" button at top of page for exporting selected/all files + +This satisfies FILE-05: User can generate tidEDA formatted files for data transmission. + + +Verify export button exists in UI and triggers download + + User can generate and download tidEDA formatted files + + + + + +- Scroll navigation works smoothly +- tidEDA export generates correct format +- Download works in browser + + + +User can navigate file list with scrolling and generate tidEDA formatted files for transmission + + + +After completion, create `.planning/phases/02-data-persistence-file-management/02-02-SUMMARY.md` + diff --git a/.planning/phases/02-data-persistence-file-management/02-RESEARCH.md b/.planning/phases/02-data-persistence-file-management/02-RESEARCH.md new file mode 100644 index 000000000..c27bc7a2d --- /dev/null +++ b/.planning/phases/02-data-persistence-file-management/02-RESEARCH.md @@ -0,0 +1,163 @@ +# Phase 2: Data Persistence & File Management - Research + +**Researched:** 2026-03-12 +**Phase:** 02-data-persistence-file-management + +## Domain Overview + +Phase 2 implements CSV data logging to flash memory with file management UI. This builds on Phase 1's Flask backend by adding: +- File system operations (list, read, delete) +- CSV file generation/management +- tidEDA formatted export +- File management UI page + +## Requirements Analysis + +| Requirement | Description | Implementation Approach | +|-------------|-------------|------------------------| +| FILE-01 | List CSV files in flash memory | Flask route returning JSON array of file metadata | +| FILE-02 | View CSV file contents | Flask route serving file content with pagination | +| FILE-03 | Navigate file list (scroll) | Frontend JavaScript with pagination or virtual scroll | +| FILE-04 | Delete CSV files | Flask DELETE endpoint with confirmation | +| FILE-05 | Generate tidEDA formatted files | Backend transformation endpoint | +| BACK-04 | File management API | All above endpoints combined | + +## Technical Implementation + +### Storage Location +- **Path:** `/myvscada/logger` (per CONTEXT.md) +- **Note:** On Pi, this may need to be relative to project or use a configurable path + +### File Naming Convention +- Format: `SP{station_id}_{timestamp}.csv` +- Example: `SP001_20260312_143022.csv` + +### API Endpoints Needed + +``` +GET /api/files - List all CSV files with metadata +GET /api/files/ - Get file contents +DELETE /api/files/ - Delete a file +POST /api/files/export - Generate tidEDA format from CSV +``` + +### Response Formats + +**GET /api/files** +```json +{ + "files": [ + { + "name": "SP001_20260312_143022.csv", + "size": 1234, + "created": "2026-03-12T14:30:22Z" + } + ], + "total": 10 +} +``` + +**GET /api/files/** +```json +{ + "name": "SP001_20260312_143022.csv", + "content": "timestamp,rainfall,solar,battery\n...", + "lines": 50 +} +``` + +### tidEDA Format + +Based on standard environmental data transmission formats. The tidEDA format should contain: +- Station identification +- Timestamp +- Sensor readings (rainfall, voltage) +- Data quality indicators + +**Suggested structure:** +``` +$STATION,SP001 +$DATETIME,2026-03-12T14:30:00Z +$DATA +timestamp,rainfall_mm,solar_v,battery_v +2026-03-12T14:30:00,0.2,12.4,12.8 +... +$END +``` + +## Frontend Implementation + +### New Page: Files (`/files`) +- Bootstrap 5.3 template (reuse existing patterns) +- File list with: + - Filename, size, date + - View button + - Delete button with confirmation + - Export to tidEDA button +- Scroll navigation (per FILE-03) +- Touch-optimized 48px+ targets + +### UI Components +- File list table/card layout +- Modal for file content viewing +- Confirmation dialogs for delete +- Loading states during API calls + +## Dependencies + +No new Python packages required beyond existing Flask: +- `os`, `datetime`, `json` (stdlib) +- Flask already installed + +## Integration Points + +1. **Main Navigation:** Add "Files" menu item +2. **Settings API reuse:** Get station ID from settings +3. **Socket.IO:** Optional - file list refresh on changes +4. **Template base:** Extend existing Bootstrap 5.3 layout + +## Validation Architecture + +**Success Criteria Validation:** + +1. User can view list of CSV files stored in flash memory + - `GET /api/files` returns file array + - Files page renders list correctly + +2. User can navigate file list (scroll up/down) + - Frontend implements scroll or pagination + - Touch targets work on 7" display + +3. User can view contents of selected CSV files + - `GET /api/files/` returns content + - Modal/displays shows formatted content + +4. User can delete unwanted CSV files from flash memory + - `DELETE /api/files/` removes file + - UI updates after deletion + +5. User can generate tidEDA formatted files + - `POST /api/files/export` generates format + - Returns downloadable content + +## Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| File path not accessible | Use configurable path, default to project data dir | +| Large files crash display | Paginate content, limit preview lines | +| Concurrent file access | Handle file locked errors gracefully | +| tidEDA format incompatibility | Define format clearly, add version field | + +## Implementation Order + +1. Backend API routes first (file operations) +2. Frontend file list page +3. File viewing modal +4. Delete functionality with confirmation +5. tidEDA export endpoint +6. Integration testing + +--- + +*Research complete - ready for planning* diff --git a/.planning/phases/02-data-persistence-file-management/02-VALIDATION.md b/.planning/phases/02-data-persistence-file-management/02-VALIDATION.md new file mode 100644 index 000000000..07cc2d9ed --- /dev/null +++ b/.planning/phases/02-data-persistence-file-management/02-VALIDATION.md @@ -0,0 +1,89 @@ +--- +phase: 02 +slug: data-persistence-file-management +status: draft +nyquist_compliant: false +wave_0_complete: false +created: 2026-03-12 +--- + +# Phase 2 — Validation Strategy + +> Per-phase validation contract for feedback sampling during execution. + +--- + +## Test Infrastructure + +| Property | Value | +|----------|-------| +| **Framework** | pytest (Python) + Flask test client | +| **Config file** | tests/ (Wave 0 creates) | +| **Quick run command** | `pytest tests/test_api.py -v` | +| **Full suite command** | `pytest tests/ -v` | +| **Estimated runtime** | ~15 seconds | + +--- + +## Sampling Rate + +- **After every task commit:** Run `pytest tests/test_api.py -v` +- **After every plan wave:** Run `pytest tests/ -v` +- **Before `/gsd-verify-work`:** Full suite must be green +- **Max feedback latency:** 30 seconds + +--- + +## Per-task Verification Map + +| task ID | Plan | Wave | Requirement | Test Type | Automated Command | File Exists | Status | +|---------|------|------|-------------|-----------|-------------------|-------------|--------| +| 02-01-01 | 01 | 1 | BACK-04 | API | `pytest tests/test_api.py::test_list_files -v` | ⚠️ W0 | ⬜ pending | +| 02-01-02 | 01 | 1 | FILE-01 | API | `pytest tests/test_api.py::test_get_file_contents -v` | ⚠️ W0 | ⬜ pending | +| 02-01-03 | 01 | 1 | FILE-04 | API | `pytest tests/test_api.py::test_delete_file -v` | ⚠️ W0 | ⬜ pending | +| 02-02-01 | 02 | 1 | FILE-02 | UI | Browser test or manual | - | ⬜ pending | +| 02-02-02 | 02 | 1 | FILE-03 | UI | Manual (touch scroll) | - | ⬜ pending | +| 02-02-03 | 02 | 2 | FILE-05 | API | `pytest tests/test_api.py::test_export_tideda -v` | ⚠️ W0 | ⬜ pending | + +*Status: ⬜ pending · ✅ green · ❌ red · ⚠️ flaky* + +--- + +## Wave 0 Requirements + +- [ ] `tests/conftest.py` — Flask test client fixture +- [ ] `tests/test_api.py` — API endpoint stubs for: + - `test_list_files` + - `test_get_file_contents` + - `test_delete_file` + - `test_export_tideda` +- [ ] `tests/__init__.py` — Package marker + +*Existing Flask test client pattern from src/app.py can be reused.* + +--- + +## Manual-Only Verifications + +| Behavior | Requirement | Why Manual | Test Instructions | +|----------|-------------|------------|-------------------| +| File list scroll navigation | FILE-03 | Touch UI interaction | Visit /files, scroll through list | +| File content display formatting | FILE-02 | Visual verification | Click file, verify content renders | +| Delete confirmation dialog | FILE-04 | UX flow | Click delete, verify confirmation | +| tidEDA export download | FILE-05 | File download | Click export, verify file downloads | + +*These require browser/UI interaction that automated tests don't reliably capture.* + +--- + +## Validation Sign-Off + +- [ ] All tasks have `` verify or Wave 0 dependencies +- [ ] Sampling continuity: no 3 consecutive tasks without automated verify +- [ ] Wave 0 covers all MISSING references +- [ ] No watch-mode flags +- [ ] Feedback latency < 30s +- [ ] `nyquist_compliant: true` set in frontmatter + +**Approval:** pending +