From 29f5721684151f891fe56f4806bce984f1a495d5 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Mar 2026 20:52:31 +0200 Subject: [PATCH] updated readme file --- .env.example | 52 +------ README.md | 196 +-------------------------- scripts/generate_example_env_file.py | 95 +++++++++++++ 3 files changed, 99 insertions(+), 244 deletions(-) create mode 100755 scripts/generate_example_env_file.py diff --git a/.env.example b/.env.example index 987f136..def7f2c 100644 --- a/.env.example +++ b/.env.example @@ -1,49 +1,3 @@ -APP_NAME=${{project.APP_NAME}} -NODE_ENV=${{project.NODE_ENV}} - -REDIS_URL=${{project.REDIS_URL}} -DATABASE_URL=${{project.DATABASE_URL}} - -COUCHBASE_CONNECTION_STRING=${{project.COUCHBASE_CONNECTION_STRING}} -COUCHBASE_USERNAME=${{project.COUCHBASE_USERNAME}} -COUCHBASE_PASSWORD=${{project.COUCHBASE_PASSWORD}} - -QDRANT_URL=${{project.QDRANT_URL}} -QDRANT_API_KEY=${{project.QDRANT_API_KEY}} - -BETTER_AUTH_SECRET=${{project.BETTER_AUTH_SECRET}} -BETTER_AUTH_URL=${{project.BETTER_AUTH_URL}} - -GOOGLE_CLIENT_ID=${{project.GOOGLE_CLIENT_ID}} -GOOGLE_CLIENT_SECRET=${{project.GOOGLE_CLIENT_SECRET}} -GOOGLE_OAUTH_SERVER_URL=${{project.GOOGLE_OAUTH_SERVER_URL}} -GOOGLE_OAUTH_SERVER_PORT=${{project.GOOGLE_OAUTH_SERVER_PORT}} - -GEMINI_API_KEY=${{project.GEMINI_API_KEY}} - -RESEND_API_KEY=${{project.RESEND_API_KEY}} -FROM_EMAIL=${{project.FROM_EMAIL}} - -INTERNAL_API_KEY="supersecretkey" - -PROCESSOR_API_URL=${{project.PROCESSOR_API_URL}} - -OPENROUTER_API_KEY=${{project.OPENROUTER_API_KEY}} -MODEL_NAME=${{project.MODEL_NAME}} - -OCR_SERVICE_URL=${{project.OCR_SERVICE_URL}} -OCR_SERVICE_TIMEOUT=${{project.OCR_SERVICE_TIMEOUT}} -OCR_SERVICE_MAX_RETRIES=${{project.OCR_SERVICE_MAX_RETRIES}} -OCR_FORCE_OCR=${{project.OCR_FORCE_OCR}} -OCR_DEBUG=${{project.OCR_DEBUG}} - -R2_BUCKET_NAME=${{project.R2_BUCKET_NAME}} -R2_REGION=${{project.R2_REGION}} -R2_ENDPOINT=${{project.R2_ENDPOINT}} -R2_ACCESS_KEY=${{project.R2_ACCESS_KEY}} -R2_SECRET_KEY=${{project.R2_SECRET_KEY}} -R2_PUBLIC_URL=${{project.R2_PUBLIC_URL}} - -MAX_FILE_SIZE=${{project.MAX_FILE_SIZE}} -ALLOWED_MIME_TYPES=${{project.ALLOWED_MIME_TYPES}} -ALLOWED_EXTENSIONS=${{project.ALLOWED_EXTENSIONS}} +APP_NAME="Illusory MAPP" +NODE_ENV="development" +LOG_LEVEL=debug diff --git a/README.md b/README.md index 9d16a83..e62a97c 100644 --- a/README.md +++ b/README.md @@ -1,197 +1,3 @@ # Illusory MAPP -Internal automation platform for aggregating device data into one backend, then running coordinated automations on top of that data. - -## Current Goal (Server-Side Stage 1) - -Build the server-side ingestion and admin visibility for: - -1. Device registration -2. Device heartbeat/ping tracking -3. SMS ingestion (ongoing) -4. Media metadata ingestion (initial sync) -5. Admin UI to inspect devices, SMS, and media - -Mobile app implementation is out of scope for now. - -## Scope and Non-Goals - -### In scope now - -- Processor endpoints that receive mobile data -- Domain logic + persistence for devices, SMS, media sync metadata -- Admin UI read views with polling for SMS (every 5 seconds) -- Observability for ingestion and read flows (logs + traces) - -### Not in scope now - -- Mobile app UX/screens or client implementation -- Automation execution workflows -- Advanced media processing pipelines - -## Delivery Plan (Actionable Tasks) - -Use this as the step-by-step implementation checklist. - -### Phase 0: Align Critical Decisions (before coding) - -- [x] Confirm device identity strategy (`deviceId` from mobile vs server-generated key). -- [x] Confirm admin ownership rule for newly registered devices (single admin user mapping). -- [x] Confirm SMS dedup strategy (client message id, hash, or `(deviceId + timestamp + sender + body)`). -- [x] Confirm media sync contract: metadata-only now vs metadata + upload references. -- [x] Confirm minimal auth for processor endpoints (shared token or signed header) for Stage 1. - -Decisions captured: - -- Keep two identifiers per device: - - `id` (server-generated primary key) - - `externalDeviceId` (sent by mobile) -- All devices are owned by the single admin user in Stage 1. -- SMS dedup uses: - - optional client message id when available - - fallback deterministic hash from `(deviceId + timestamp + sender + body)`. -- Media sync is metadata plus upload reference (`fileId`) to the file domain/R2 object. -- Stage 1 endpoint auth: - - register is open in trusted network - - ping/sync endpoints must include device id header; request is allowed only if device exists. - -### Phase 1: Data Model and DB Migrations - -Target: `packages/db/schema/*` + new migration files. - -- [x] Add `mobile_device` table: - - Fields: `id`, `externalDeviceId`, `name`, `manufacturer`, `model`, `androidVersion`, `ownerUserId`, `lastPingAt`, `createdAt`, `updatedAt`. -- [x] Add `mobile_sms` table: - - Fields: `id`, `deviceId`, `externalMessageId?`, `sender`, `recipient?`, `body`, `sentAt`, `receivedAt?`, `rawPayload?`, `createdAt`. -- [x] Add `mobile_media_asset` table: - - Fields: `id`, `deviceId`, `externalMediaId?`, `mimeType`, `filename?`, `capturedAt?`, `sizeBytes?`, `hash?`, `metadata`, `createdAt`. -- [x] Add indexes for common reads: - - `mobile_device.lastPingAt` - - `mobile_sms.deviceId + sentAt desc` - - `mobile_media_asset.deviceId + createdAt desc` -- [x] Export schema from `packages/db/schema/index.ts`. - -Definition of done: - -- [x] Tables and indexes exist in schema + migration files. -- [x] Naming matches existing conventions. - -### Phase 2: Logic Domain (`@pkg/logic`) - -Target: `packages/logic/domains/mobile/*` (new domain). - -- [x] Create `data.ts` with Valibot schemas and inferred types for: - - register device payload - - ping payload - - sms sync payload - - media sync payload - - admin query filters/pagination -- [x] Create `errors.ts` with domain error constructors using `getError`. -- [x] Create `repository.ts` with ResultAsync operations for: - - upsert device - - update last ping - - bulk insert sms (idempotent) - - bulk insert media metadata (idempotent) - - list devices - - get device detail - - list sms by device (paginated, latest first) - - list media by device (paginated, latest first) - - delete single media asset - - delete device (removed all child sms, and media assets, and by extensionn the associated files in r2+db as well) -- [x] Create `controller.ts` as use-case layer. -- [x] Wrap repository/controller operations with `traceResultAsync` and include `fctx`. - -Definition of done: - -- [x] Processor and main app can consume this domain without direct DB usage. -- [x] No ad-hoc thrown errors in domain flow; Result pattern is preserved. - -### Phase 3: Processor Ingestion API (`apps/processor`) - -Target: `apps/processor/src/domains/mobile/router.ts`. - -- [x] Replace stub endpoints with full handlers: - - `POST /register` - - `PUT /ping` - - `PUT /sms/sync` - - `PUT /media/sync` -- [x] Validate request body using schemas from `@pkg/logic/domains/mobile/data`. -- [x] Build `FlowExecCtx` per request (flow id always; user/session when available). -- [x] Call mobile controller methods; return `{ data, error }` response shape. -- [x] Add basic endpoint protection agreed in Phase 0. -- [x] Add request-level structured logging for success/failure. - -Definition of done: - -- [x] Endpoints persist data into new tables. -- [x] Endpoint failures return normalized domain errors. - -### Phase 4: Admin UI in `apps/main` - -Target: - -- `apps/main/src/lib/domains/mobile/*` (new) -- `apps/main/src/routes/(main)/devices/[deviceId]` (new) - -- [x] Add remote functions: - - `getDevicesSQ` - - `getDeviceDetailSQ` - - `getDeviceSmsSQ` - - `getDeviceMediaSQ` -- [x] Add VM (`devices.vm.svelte.ts`) that: - - fetches devices list - - fetches selected device detail - - polls SMS every 5 seconds while device view is open - - handles loading/error states -- [x] Add UI pages/components: - - `/dashboard` list with device identity + last ping - - `/devices/[deviceId]` detail with tabs: - - Device info - - SMS feed - - Media assets list -- [x] Add sidebar/navigation entry for Devices. - -Definition of done: - -- [x] Admin can browse devices and open each device detail. -- [x] SMS view refreshes every 5 seconds and shows new data. - -### Phase 5: Observability Stage 1 - -Targets: - -- `packages/logic/core/observability.ts` (use existing helpers) -- Processor/mobile domain handlers and repository/controller paths - -- [x] Add span names for key flows: - - `mobile.register` - - `mobile.ping` - - `mobile.sms.sync` - - `mobile.media.sync` - - `mobile.devices.list` - - `mobile.device.detail` -- [x] Add structured domain events with device id, counts, durations. -- [x] Ensure errors include `flowId` consistently. - -Definition of done: - -- [x] Can trace one request from processor endpoint to DB operation via shared `flowId`. - -### Phase 6: Handoff Readiness (Team Test Phase) - -- [x] Prepare endpoint payload examples for mobile team (in a `spec.mobile.md` file). -- [x] Provide a short operator checklist: - - register device - - verify ping updates - - verify sms appears in admin - - verify media appears in admin - -## Suggested Build Order - -1. Phase 0 -2. Phase 1 -3. Phase 2 -4. Phase 3 -5. Phase 4 -6. Phase 5 -7. Phase 6 +Illusory MAPP is an internal project focused on building the foundation for future automation workflows. Right now, the platform provides a working ingestion backend and a simple admin dashboard: devices can register and sync data, and the dashboard lets the team inspect each device’s current state, including SMS history and uploaded gallery/media assets. This gives us a stable operational base for visibility and data quality before layering on higher-level automation features. diff --git a/scripts/generate_example_env_file.py b/scripts/generate_example_env_file.py new file mode 100755 index 0000000..49bf0ee --- /dev/null +++ b/scripts/generate_example_env_file.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +ENV_ASSIGNMENT_RE = re.compile( + r"^(\s*(?:export\s+)?)([A-Za-z_][A-Za-z0-9_]*)(\s*=\s*)(.*)$" +) + + +def split_inline_comment(rhs: str) -> tuple[str, str]: + in_single = False + in_double = False + escaped = False + + for i, char in enumerate(rhs): + if escaped: + escaped = False + continue + + if char == "\\": + escaped = True + continue + + if char == "'" and not in_double: + in_single = not in_single + continue + + if char == '"' and not in_single: + in_double = not in_double + continue + + if char == "#" and not in_single and not in_double: + if i == 0 or rhs[i - 1].isspace(): + return rhs[:i], rhs[i:] + + return rhs, "" + + +def transform_line(line: str) -> str: + stripped = line.strip() + if stripped == "" or stripped.startswith("#"): + return line + + newline = "" + raw = line + if line.endswith("\r\n"): + newline = "\r\n" + raw = line[:-2] + elif line.endswith("\n"): + newline = "\n" + raw = line[:-1] + + match = ENV_ASSIGNMENT_RE.match(raw) + if not match: + return line + + prefix, key, delimiter, rhs = match.groups() + value_part, comment_part = split_inline_comment(rhs) + trailing_ws = value_part[len(value_part.rstrip()) :] + placeholder = f"${{{{project.{key}}}}}" + + return f"{prefix}{key}{delimiter}{placeholder}{trailing_ws}{comment_part}{newline}" + + +def generate_example_env(source_path: Path, target_path: Path) -> None: + lines = source_path.read_text(encoding="utf-8").splitlines(keepends=True) + transformed = [transform_line(line) for line in lines] + target_path.write_text("".join(transformed), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate .env.example using ${{project.KEY}} placeholders." + ) + parser.add_argument("--source", default=".env", help="Path to source .env file") + parser.add_argument( + "--target", default=".env.example", help="Path to output .env.example file" + ) + args = parser.parse_args() + + source_path = Path(args.source) + target_path = Path(args.target) + + if not source_path.exists(): + raise FileNotFoundError(f"Source env file not found: {source_path}") + + generate_example_env(source_path, target_path) + + +if __name__ == "__main__": + main()