From 14c974f54c08c5c9f0a805db2930f6505e928dd2 Mon Sep 17 00:00:00 2001 From: 4NK Date: Fri, 3 Apr 2026 22:35:57 +0200 Subject: [PATCH] Add smart-ide-tools-bridge API for submodule tools + central local env - New service: tools bridge (port 37147) registry + Carbonyl/PageIndex/Chandra POST jobs - config/services.local.env.example and gitignore for services.local.env - .env.example for repos-devtools, regex-search, ia-dev-gateway, orchestrator, claw proxy, langextract - Orchestrator intents: tools.registry, tools.carbonyl.plan, tools.pageindex.run, tools.chandra.ocr - Docs: API + repo service fiche, architecture index; do not commit dist/ --- .gitignore | 3 + config/README.md | 3 + config/services.local.env.example | 65 +++++ docs/API/README.md | 1 + docs/API/orchestrator.md | 2 +- docs/API/smart-ide-tools-bridge-api.md | 42 +++ docs/README.md | 8 + docs/features/orchestrator-api.md | 2 + docs/repo/README.md | 1 + docs/repo/service-carbonyl.md | 3 +- docs/repo/service-chandra.md | 3 +- docs/repo/service-pageindex.md | 3 +- docs/repo/service-smart-ide-tools-bridge.md | 18 ++ docs/services.md | 6 +- docs/system-architecture.md | 2 + services/agent-regex-search-api/.env.example | 4 + services/carbonyl/README.md | 2 + services/chandra/README.md | 2 + services/claw-harness-api/proxy/.env.example | 4 + services/ia-dev-gateway/.env.example | 4 + services/langextract-api/.env.example | 4 + services/pageindex/README.md | 2 + services/repos-devtools-server/.env.example | 5 + services/smart-ide-orchestrator/.env.example | 12 + services/smart-ide-orchestrator/src/server.ts | 47 ++++ services/smart-ide-tools-bridge/.env.example | 14 + services/smart-ide-tools-bridge/.gitignore | 2 + services/smart-ide-tools-bridge/README.md | 25 ++ .../smart-ide-tools-bridge/package-lock.json | 51 ++++ services/smart-ide-tools-bridge/package.json | 20 ++ services/smart-ide-tools-bridge/src/auth.ts | 21 ++ .../smart-ide-tools-bridge/src/httpUtil.ts | 13 + services/smart-ide-tools-bridge/src/paths.ts | 49 ++++ services/smart-ide-tools-bridge/src/server.ts | 261 ++++++++++++++++++ .../smart-ide-tools-bridge/src/spawnJob.ts | 35 +++ services/smart-ide-tools-bridge/tsconfig.json | 16 ++ 36 files changed, 750 insertions(+), 5 deletions(-) create mode 100644 config/README.md create mode 100644 config/services.local.env.example create mode 100644 docs/API/smart-ide-tools-bridge-api.md create mode 100644 docs/repo/service-smart-ide-tools-bridge.md create mode 100644 services/agent-regex-search-api/.env.example create mode 100644 services/claw-harness-api/proxy/.env.example create mode 100644 services/ia-dev-gateway/.env.example create mode 100644 services/langextract-api/.env.example create mode 100644 services/repos-devtools-server/.env.example create mode 100644 services/smart-ide-orchestrator/.env.example create mode 100644 services/smart-ide-tools-bridge/.env.example create mode 100644 services/smart-ide-tools-bridge/.gitignore create mode 100644 services/smart-ide-tools-bridge/README.md create mode 100644 services/smart-ide-tools-bridge/package-lock.json create mode 100644 services/smart-ide-tools-bridge/package.json create mode 100644 services/smart-ide-tools-bridge/src/auth.ts create mode 100644 services/smart-ide-tools-bridge/src/httpUtil.ts create mode 100644 services/smart-ide-tools-bridge/src/paths.ts create mode 100644 services/smart-ide-tools-bridge/src/server.ts create mode 100644 services/smart-ide-tools-bridge/src/spawnJob.ts create mode 100644 services/smart-ide-tools-bridge/tsconfig.json diff --git a/.gitignore b/.gitignore index 4a66620..68e4d7d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ services/docv/target/ # Surcharges locales pull-sync (cron) cron/config.local.env +# Configuration locale agrégée des services (copie de config/services.local.env.example) +config/services.local.env + # logs/ : le répertoire reste versionné (README.md, logs/.gitignore) ; aucun fichier journal versionné logs/**/*.log diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..db9712e --- /dev/null +++ b/config/README.md @@ -0,0 +1,3 @@ +# Configuration locale + +- **`services.local.env.example`** : variables pour l’IDE et tous les services HTTP ; copier vers **`services.local.env`** (non versionné, voir racine **`.gitignore`**). diff --git a/config/services.local.env.example b/config/services.local.env.example new file mode 100644 index 0000000..97c7e60 --- /dev/null +++ b/config/services.local.env.example @@ -0,0 +1,65 @@ +# Local IDE / agent configuration for all smart_ide HTTP services and the tools bridge. +# Copy to config/services.local.env and fill secrets. Do not commit services.local.env. +# +# Load before starting processes, e.g.: set -a && source config/services.local.env && set +a + +# Monorepo root (path validation for tools-bridge jobs) +SMART_IDE_MONOREPO_ROOT=/absolute/path/to/smart_ide + +# Optional: extra allowed path prefixes (comma-separated) for tools-bridge file jobs +# TOOLS_ALLOWED_PATH_PREFIXES=/data/clones,/home/user/projects + +# --- smart-ide-tools-bridge (Carbonyl / PageIndex / Chandra + registry) --- +TOOLS_BRIDGE_HOST=127.0.0.1 +TOOLS_BRIDGE_PORT=37147 +TOOLS_BRIDGE_TOKEN= +TOOLS_BRIDGE_JOB_TIMEOUT_MS=3600000 + +# --- Core HTTP micro-services --- +REPOS_DEVTOOLS_HOST=127.0.0.1 +REPOS_DEVTOOLS_PORT=37140 +REPOS_DEVTOOLS_TOKEN= +REPOS_DEVTOOLS_ROOT= + +LANGEXTRACT_API_HOST=127.0.0.1 +LANGEXTRACT_API_PORT=37141 +LANGEXTRACT_SERVICE_TOKEN= + +CLAW_PROXY_HOST=127.0.0.1 +CLAW_PROXY_PORT=37142 +CLAW_PROXY_TOKEN= +CLAW_UPSTREAM_URL= + +REGEX_SEARCH_HOST=127.0.0.1 +REGEX_SEARCH_PORT=37143 +REGEX_SEARCH_TOKEN= +REGEX_SEARCH_ROOT= + +LOCAL_OFFICE_URL=http://127.0.0.1:8000 +LOCAL_OFFICE_API_KEY= + +IA_DEV_GATEWAY_HOST=127.0.0.1 +IA_DEV_GATEWAY_PORT=37144 +IA_DEV_GATEWAY_TOKEN= + +ORCHESTRATOR_HOST=127.0.0.1 +ORCHESTRATOR_PORT=37145 +ORCHESTRATOR_TOKEN= + +ANYTHINGLLM_DEVTOOLS_HOST=127.0.0.1 +ANYTHINGLLM_DEVTOOLS_PORT=37146 +ANYTHINGLLM_DEVTOOLS_TOKEN= +ANYTHINGLLM_BASE_URL= +ANYTHINGLLM_API_KEY= +REPOS_DEVTOOLS_URL=http://127.0.0.1:37140 + +# Orchestrator → tools bridge (for intent resolution hints) +TOOLS_BRIDGE_URL=http://127.0.0.1:37147 + +# Ollama / AnythingLLM (orchestrator) +OLLAMA_URL=http://127.0.0.1:11434 +ANYTHINGLLM_BASE_URL= + +# Carbonyl (docker / native — used by bridge open-plan response) +CARBONYL_DOCKER_IMAGE=fathyb/carbonyl +CARBONYL_RUNNER=docker diff --git a/docs/API/README.md b/docs/API/README.md index f515b3e..6015c90 100644 --- a/docs/API/README.md +++ b/docs/API/README.md @@ -12,6 +12,7 @@ Documentation des **API HTTP** exposées par les services sous [`services/`](../ | **ia-dev-gateway** | Bearer | `37144` (spécification) | [ia-dev-gateway.md](./ia-dev-gateway.md) | | **smart_ide-orchestrator** | Bearer (spécification) | `37145` (spécification) | [orchestrator.md](./orchestrator.md) | | **anythingllm-devtools** | Bearer | `37146` | [anythingllm-devtools-api.md](./anythingllm-devtools-api.md) | +| **smart-ide-tools-bridge** | Bearer (sauf `/health`) | `37147` | [smart-ide-tools-bridge-api.md](./smart-ide-tools-bridge-api.md) | | **docv** (externe) | selon dépôt Enso | selon déploiement | [docv.md](./docv.md) | **OpenAPI** : FastAPI expose une spec interactive pour **langextract-api** (`/docs`) et **local-office** (`/docs`) une fois le service démarré. diff --git a/docs/API/orchestrator.md b/docs/API/orchestrator.md index 972725b..895966b 100644 --- a/docs/API/orchestrator.md +++ b/docs/API/orchestrator.md @@ -14,7 +14,7 @@ Résout une intention sans nécessairement l’exécuter (si `dryRun: true`). | Champ | Obligatoire | Description | |-------|-------------|-------------| -| `intent` | oui | Identifiant stable (`code.complete`, `rag.query`, `agent.run`, …) | +| `intent` | oui | Identifiant stable (`code.complete`, `rag.query`, `agent.run`, `tools.registry`, `tools.pageindex.run`, …) | | `context` | non | Objet libre (fichiers ouverts, sélection, etc.) | | `projectId` | non | Projet `ia_dev` / workspace | | `env` | non | `test` \| `pprod` \| `prod` | diff --git a/docs/API/smart-ide-tools-bridge-api.md b/docs/API/smart-ide-tools-bridge-api.md new file mode 100644 index 0000000..ff376d9 --- /dev/null +++ b/docs/API/smart-ide-tools-bridge-api.md @@ -0,0 +1,42 @@ +# API smart-ide-tools-bridge + +Écoute **`127.0.0.1`** par défaut, port **`37147`** (`TOOLS_BRIDGE_PORT`). Auth : **`Authorization: Bearer `** (obligatoire sauf **`GET /health`**). + +## Endpoints + +| Méthode | Chemin | Rôle | +|---------|--------|------| +| GET | `/health` | Santé (sans Bearer). | +| GET | `/v1/registry` | JSON : URLs des services HTTP du monorepo + métadonnées des outils sous-module (chemins relatifs). | +| POST | `/v1/carbonyl/open-plan` | Corps `{"url":"https://…"}` → plan d’exécution terminal (`docker` ou `native`). | +| POST | `/v1/pageindex/run` | Corps `{"mode":"pdf"\|"md","inputPath":"/abs/…"}` → sous-processus `run-pageindex.sh` ; réponse inclut `resultPath` si `…/upstream/results/_structure.json` existe. | +| POST | `/v1/chandra/ocr` | Corps `{"inputPath":"…","outputPath":"…","method":"hf"\|"vllm"}` (défaut `hf`) → scripts Chandra. | + +## Corps JSON (exemples) + +**Carbonyl** + +```json +{ "url": "https://example.com" } +``` + +**PageIndex** + +```json +{ "mode": "pdf", "inputPath": "/abs/smart_ide/docs/foo.pdf" } +``` + +**Chandra** + +```json +{ "inputPath": "/abs/in.pdf", "outputPath": "/abs/out_dir", "method": "hf" } +``` + +## Sécurité des chemins + +`inputPath` / `outputPath` doivent résoudre sous **`SMART_IDE_MONOREPO_ROOT`** ou sous un préfixe listé dans **`TOOLS_ALLOWED_PATH_PREFIXES`** (séparateur virgule). + +## Voir aussi + +- [README.md](../README.md) (table des ports) +- [repo/service-smart-ide-tools-bridge.md](../repo/service-smart-ide-tools-bridge.md) diff --git a/docs/README.md b/docs/README.md index fbb1973..0e83da4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,13 @@ Index principal. Les **fonctionnalités** détaillées sont dans [`features/`](./features/). La **documentation opérationnelle** regroupée (anciens `README.md` à la racine et sous `cron/`, `projects/`, `services/`, etc.) est dans **[`repo/`](./repo/)**. +## Configuration locale agrégée + +| Fichier | Rôle | +|---------|------| +| [config/services.local.env.example](../config/services.local.env.example) | Ports, jetons, `SMART_IDE_MONOREPO_ROOT` — copier vers `config/services.local.env` | +| [config/README.md](../config/README.md) | Rôle du répertoire `config/` | + ## Documentation du dépôt (`repo/`) Vue d’ensemble et index complet : **[repo/README.md](./repo/README.md)**. Règles/agents IDE : **[repo/smartide-config-directory.md](./repo/smartide-config-directory.md)**. @@ -25,6 +32,7 @@ Vue d’ensemble et index complet : **[repo/README.md](./repo/README.md)**. Règ | [repo/service-anythingllm-devtools.md](./repo/service-anythingllm-devtools.md) | Service HTTP AnythingLLM + devtools | | [repo/service-carbonyl.md](./repo/service-carbonyl.md) | Carbonyl (navigateur terminal), prévisualisation test | | [repo/service-pageindex.md](./repo/service-pageindex.md) | PageIndex (index vectorless, définition sémantique documents) | +| [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md) | Pont HTTP IDE + sous-modules CLI | | [repo/service-chandra.md](./repo/service-chandra.md) | Chandra OCR (PDF / images structurés) | | [repo/extension-anythingllm-workspaces.md](./repo/extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; voir anythingllm-devtools) | diff --git a/docs/features/orchestrator-api.md b/docs/features/orchestrator-api.md index 98e1c42..dd70bd9 100644 --- a/docs/features/orchestrator-api.md +++ b/docs/features/orchestrator-api.md @@ -37,6 +37,7 @@ La référence OpenAPI détaillée : [API/orchestrator.md](../API/orchestrator.m | `extract.entities` | langextract-api | | `doc.office.upload` | local-office (`X-API-Key`) | | `agent.run`, `deploy.trigger` | ia-dev-gateway | +| `tools.registry`, `tools.carbonyl.plan`, `tools.pageindex.run`, `tools.chandra.ocr` | smart-ide-tools-bridge (`TOOLS_BRIDGE_URL`, jeton `TOOLS_BRIDGE_TOKEN`) | ## Variables d’environnement (cible) @@ -49,6 +50,7 @@ La référence OpenAPI détaillée : [API/orchestrator.md](../API/orchestrator.m | `ANYTHINGLLM_API_KEY` | Clé API | | `REPOS_DEVTOOLS_URL`, `REPOS_DEVTOOLS_TOKEN` | … | | `IA_DEV_GATEWAY_URL`, `IA_DEV_GATEWAY_TOKEN` | … | +| `TOOLS_BRIDGE_URL` | Base URL du pont IDE (défaut `http://127.0.0.1:37147`) | Les valeurs diffèrent par **environnement** (test / pprod / prod) — voir [platform-target.md](../platform-target.md). diff --git a/docs/repo/README.md b/docs/repo/README.md index f246dc3..256dd52 100644 --- a/docs/repo/README.md +++ b/docs/repo/README.md @@ -40,6 +40,7 @@ Toute la documentation **opérationnelle** qui vivait auparavant sous des `READM | [service-carbonyl.md](./service-carbonyl.md) | Carbonyl (navigateur terminal), sous-module amont | | [service-pageindex.md](./service-pageindex.md) | PageIndex (index sémantique vectorless), sous-module amont | | [service-chandra.md](./service-chandra.md) | Chandra OCR, sous-module amont | +| [service-smart-ide-tools-bridge.md](./service-smart-ide-tools-bridge.md) | Pont HTTP IDE + outils sous-modules | | [extension-anythingllm-workspaces.md](./extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; anythingllm-devtools) | Les **spécifications** détaillées (contrats HTTP, sécurité, orchestration) restent dans [../API/README.md](../API/README.md) et [../features/](../features/). diff --git a/docs/repo/service-carbonyl.md b/docs/repo/service-carbonyl.md index c9e62ce..583a9ce 100644 --- a/docs/repo/service-carbonyl.md +++ b/docs/repo/service-carbonyl.md @@ -6,8 +6,9 @@ Navigateur **terminal** basé sur Chromium — amont **[fathyb/carbonyl](https:/ - **Pilotage / visualisation** d’URLs dans un terminal (SSH sans GUI, session locale). - **Prévisualisation** des applications déployées en **test** : URL optionnelle dans **`projects//conf.json`** → **`smart_ide.preview_urls.test`**, ouverte par **`scripts/open-carbonyl-preview-test.sh`**. +- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/carbonyl/open-plan` (retourne la commande à lancer en terminal) — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md). -Ce n’est **pas** un service HTTP : pas de port d’écoute dans smart_ide. L’exécution est un **processus interactif** (Docker ou binaire `carbonyl`). +Carbonyl lui-même reste un **processus interactif** (Docker ou binaire `carbonyl`), pas un listener HTTP dédié. ## Exploitation diff --git a/docs/repo/service-chandra.md b/docs/repo/service-chandra.md index f455ff2..bfbb720 100644 --- a/docs/repo/service-chandra.md +++ b/docs/repo/service-chandra.md @@ -5,7 +5,8 @@ OCR et extraction **structurée** (PDF / images → Markdown, HTML, JSON avec mi ## Rôle dans smart_ide - **Numérisation** de documents complexes (tableaux, formulaires, manuscrits, math). -- **Pas de listener HTTP** dans ce dépôt : CLI **`chandra`**, lancée par **`services/chandra/run-chandra.sh`** après installation dans **`upstream/`** (`uv sync` ou équivalent). +- **CLI** : **`services/chandra/run-chandra.sh`** / **`run-chandra-hf.sh`** après installation dans **`upstream/`**. +- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/chandra/ocr` — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md). ## Licences diff --git a/docs/repo/service-pageindex.md b/docs/repo/service-pageindex.md index c08c76e..a87f93c 100644 --- a/docs/repo/service-pageindex.md +++ b/docs/repo/service-pageindex.md @@ -5,7 +5,8 @@ Indexation **sémantique structurée** de documents longs (PDF, Markdown) via le ## Rôle dans smart_ide - **Pilotage de la définition sémantique** des documents : produire ou exploiter un **index arborescent** traçable (titres, pages, nœuds), distinct du RAG par embeddings **AnythingLLM**. -- **Pas de service HTTP** dans ce dépôt : exécution **CLI** Python sous **`services/pageindex/upstream/`**, lancée via **`services/pageindex/run-pageindex.sh`**. +- **CLI** : **`services/pageindex/run-pageindex.sh`** (Python sous **`upstream/`**). +- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/pageindex/run` — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md). ## Exploitation diff --git a/docs/repo/service-smart-ide-tools-bridge.md b/docs/repo/service-smart-ide-tools-bridge.md new file mode 100644 index 0000000..94d7df4 --- /dev/null +++ b/docs/repo/service-smart-ide-tools-bridge.md @@ -0,0 +1,18 @@ +# Service smart-ide-tools-bridge (`services/smart-ide-tools-bridge/`) + +Pont HTTP pour l’**IDE** et les agents : exposition d’une **API** au-dessus des outils **CLI** (sous-modules **Carbonyl**, **PageIndex**, **Chandra**) et **registre** des autres services locaux (URLs + noms de variables de jeton, sans secrets). + +## Configuration locale + +- Fichier agrégé : **[`config/services.local.env.example`](../../config/services.local.env.example)** → copier en **`config/services.local.env`** (gitignoré). +- Service : **`services/smart-ide-tools-bridge/.env.example`** (`TOOLS_BRIDGE_TOKEN`, `SMART_IDE_MONOREPO_ROOT`, `TOOLS_ALLOWED_PATH_PREFIXES`, délais). + +## Exploitation + +Voir **[`services/smart-ide-tools-bridge/README.md`](../../services/smart-ide-tools-bridge/README.md)** et **[API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md)**. + +**Orchestrateur** : intents `tools.registry`, `tools.carbonyl.plan`, `tools.pageindex.run`, `tools.chandra.ocr` — variable **`TOOLS_BRIDGE_URL`**. + +## Voir aussi + +- [service-carbonyl.md](./service-carbonyl.md), [service-pageindex.md](./service-pageindex.md), [service-chandra.md](./service-chandra.md) diff --git a/docs/services.md b/docs/services.md index 985803d..3db8d39 100644 --- a/docs/services.md +++ b/docs/services.md @@ -38,7 +38,11 @@ Services d’appoint sur **`127.0.0.1`** (souvent auth **Bearer**) : Git devtool **PageIndex** (`services/pageindex/`) n’est pas un listener HTTP : outil Python (sous-module [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex)) pour produire un **index arborescent** sémantique sur PDF / Markdown, en complément du RAG **AnythingLLM** — [repo/service-pageindex.md](./repo/service-pageindex.md). -**Chandra OCR** (`services/chandra/`) n’est pas un listener HTTP : CLI (sous-module [datalab-to/chandra](https://github.com/datalab-to/chandra)) pour **OCR** PDF / images vers Markdown, HTML, JSON avec layout — [repo/service-chandra.md](./repo/service-chandra.md). +**Chandra OCR** (`services/chandra/`) n’est pas un listener HTTP : CLI (sous-module [datalab-to/chandra](https://github.com/datalab-to/chandra)) pour **OCR** PDF / images vers Markdown, HTML, JSON avec layout — [repo/service-chandra.md](./repo/service-chandra.md). L’**IDE** peut piloter Chandra via **`services/smart-ide-tools-bridge/`** — [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md). + +**smart-ide-tools-bridge** (`services/smart-ide-tools-bridge/`) : **API** locale (registre + Carbonyl / PageIndex / Chandra) — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md). + +**Configuration locale** : [config/services.local.env.example](../config/services.local.env.example). ## Documentation liée diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 8d552bc..3c60113 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -44,6 +44,8 @@ Conséquences : | `ia_dev/` | Agents, déploiements — exécution sous policy ; `ia_dev/projects/` peut pointer vers `../../projects/` (lien) ; voir [ia_dev-module.md](./ia_dev-module.md) | | `services/ia-dev-gateway/` | Gateway HTTP (stub runner) : registre agents `.md`, runs, SSE — [features/ia-dev-service.md](./features/ia-dev-service.md) | | `services/smart-ide-orchestrator/` | Routage intentions (stub forward) — [features/orchestrator-api.md](./features/orchestrator-api.md) | +| `services/smart-ide-tools-bridge/` | API IDE : registre des services + Carbonyl / PageIndex / Chandra — [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md) | +| `config/` | Configuration locale agrégée (`services.local.env.example`) pour l’IDE et les services | ## Environnements test, pprod, prod diff --git a/services/agent-regex-search-api/.env.example b/services/agent-regex-search-api/.env.example new file mode 100644 index 0000000..8635659 --- /dev/null +++ b/services/agent-regex-search-api/.env.example @@ -0,0 +1,4 @@ +REGEX_SEARCH_HOST=127.0.0.1 +REGEX_SEARCH_PORT=37143 +REGEX_SEARCH_TOKEN= +REGEX_SEARCH_ROOT= diff --git a/services/carbonyl/README.md b/services/carbonyl/README.md index c4acc07..06bdd3a 100644 --- a/services/carbonyl/README.md +++ b/services/carbonyl/README.md @@ -38,6 +38,8 @@ Dans **`projects//conf.json`**, sous **`smart_ide`**, champ optionnel **`pre Le script **`scripts/open-carbonyl-preview-test.sh`** lit **`projects/active-project.json`** (ou **`SMART_IDE_PROJECT_ID`**, ou argument **`--project `**) puis **`preview_urls.test`**. En secours : variable **`PREVIEW_TEST_URL`**. +**API IDE** : plan d’ouverture d’URL via **`smart-ide-tools-bridge`** — `POST /v1/carbonyl/open-plan` (Bearer) — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md). + Documentation : [docs/repo/service-carbonyl.md](../../docs/repo/service-carbonyl.md), [docs/features/carbonyl-terminal-browser.md](../../docs/features/carbonyl-terminal-browser.md). ## Licence amont diff --git a/services/chandra/README.md b/services/chandra/README.md index 22111ac..56ac7f5 100644 --- a/services/chandra/README.md +++ b/services/chandra/README.md @@ -55,6 +55,8 @@ cd .. - **OCR / numérisation structurée** pour pipelines documentaires, en amont de **PageIndex** ([PageIndex](../pageindex/README.md)) ou d’**AnythingLLM** / **docv**. - **Pas de service HTTP** dans ce dépôt : exécution **CLI** (comme **`services/pageindex/`**). +**API IDE** : OCR via **`smart-ide-tools-bridge`** — `POST /v1/chandra/ocr` — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md). + Documentation : [docs/repo/service-chandra.md](../../docs/repo/service-chandra.md), [docs/features/chandra-ocr-documents.md](../../docs/features/chandra-ocr-documents.md). ## Ressources amont diff --git a/services/claw-harness-api/proxy/.env.example b/services/claw-harness-api/proxy/.env.example new file mode 100644 index 0000000..56b69ad --- /dev/null +++ b/services/claw-harness-api/proxy/.env.example @@ -0,0 +1,4 @@ +CLAW_PROXY_HOST=127.0.0.1 +CLAW_PROXY_PORT=37142 +CLAW_PROXY_TOKEN= +CLAW_UPSTREAM_URL= diff --git a/services/ia-dev-gateway/.env.example b/services/ia-dev-gateway/.env.example new file mode 100644 index 0000000..3cdcae4 --- /dev/null +++ b/services/ia-dev-gateway/.env.example @@ -0,0 +1,4 @@ +IA_DEV_GATEWAY_HOST=127.0.0.1 +IA_DEV_GATEWAY_PORT=37144 +IA_DEV_GATEWAY_TOKEN= +IA_DEV_ROOT= diff --git a/services/langextract-api/.env.example b/services/langextract-api/.env.example new file mode 100644 index 0000000..d6832b2 --- /dev/null +++ b/services/langextract-api/.env.example @@ -0,0 +1,4 @@ +LANGEXTRACT_API_HOST=127.0.0.1 +LANGEXTRACT_API_PORT=37141 +LANGEXTRACT_SERVICE_TOKEN= +LANGEXTRACT_API_KEY= diff --git a/services/pageindex/README.md b/services/pageindex/README.md index a2716d5..d7e5a36 100644 --- a/services/pageindex/README.md +++ b/services/pageindex/README.md @@ -41,6 +41,8 @@ Les options (`--model`, `--toc-check-pages`, etc.) sont celles documentées dans - **Définition sémantique structurée** des documents (arbre de sections, résumés de nœuds) pour outillage, agents ou pipelines **hors** AnythingLLM vectoriel. - Complément possible à la mémoire documentaire **AnythingLLM** ([anythingllm-workspaces.md](../../docs/anythingllm-workspaces.md)) : PageIndex ne remplace pas l’ingestion RAG classique ; il fournit un **index explicable** pour navigation et raisonnement. +**API IDE** : exécution index via **`smart-ide-tools-bridge`** — `POST /v1/pageindex/run` — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md). + Documentation : [docs/repo/service-pageindex.md](../../docs/repo/service-pageindex.md), [docs/features/pageindex-semantic-documents.md](../../docs/features/pageindex-semantic-documents.md). ## Ressources amont diff --git a/services/repos-devtools-server/.env.example b/services/repos-devtools-server/.env.example new file mode 100644 index 0000000..6eb6940 --- /dev/null +++ b/services/repos-devtools-server/.env.example @@ -0,0 +1,5 @@ +REPOS_DEVTOOLS_HOST=127.0.0.1 +REPOS_DEVTOOLS_PORT=37140 +REPOS_DEVTOOLS_TOKEN= +# Git clone root (must exist) +REPOS_DEVTOOLS_ROOT= diff --git a/services/smart-ide-orchestrator/.env.example b/services/smart-ide-orchestrator/.env.example new file mode 100644 index 0000000..f63d7fe --- /dev/null +++ b/services/smart-ide-orchestrator/.env.example @@ -0,0 +1,12 @@ +ORCHESTRATOR_HOST=127.0.0.1 +ORCHESTRATOR_PORT=37145 +ORCHESTRATOR_TOKEN= + +OLLAMA_URL=http://127.0.0.1:11434 +ANYTHINGLLM_BASE_URL= +REPOS_DEVTOOLS_URL=http://127.0.0.1:37140 +REGEX_SEARCH_URL=http://127.0.0.1:37143 +LANGEXTRACT_URL=http://127.0.0.1:37141 +LOCAL_OFFICE_URL=http://127.0.0.1:8000 +IA_DEV_GATEWAY_URL=http://127.0.0.1:37144 +TOOLS_BRIDGE_URL=http://127.0.0.1:37147 diff --git a/services/smart-ide-orchestrator/src/server.ts b/services/smart-ide-orchestrator/src/server.ts index 1775b71..eaf2cf6 100644 --- a/services/smart-ide-orchestrator/src/server.ts +++ b/services/smart-ide-orchestrator/src/server.ts @@ -36,6 +36,9 @@ const localOfficeUrl = (): string => const iaDevGatewayUrl = (): string => (process.env.IA_DEV_GATEWAY_URL ?? "http://127.0.0.1:37144").replace(/\/+$/, ""); +const toolsBridgeUrl = (): string => + (process.env.TOOLS_BRIDGE_URL ?? "http://127.0.0.1:37147").replace(/\/+$/, ""); + const resolveIntent = (intent: string): Resolution => { switch (intent) { case "code.complete": @@ -94,6 +97,50 @@ const resolveIntent = (intent: string): Resolution => { action: "post_run", upstream: { method: "POST", url: `${iaDevGatewayUrl()}/v1/runs`, headersHint: ["Authorization", "Content-Type"] }, }; + case "tools.registry": + return { + resolved: true, + target: "service", + action: "tools_bridge_registry", + upstream: { + method: "GET", + url: `${toolsBridgeUrl()}/v1/registry`, + headersHint: ["Authorization"], + }, + }; + case "tools.carbonyl.plan": + return { + resolved: true, + target: "service", + action: "tools_carbonyl_open_plan", + upstream: { + method: "POST", + url: `${toolsBridgeUrl()}/v1/carbonyl/open-plan`, + headersHint: ["Authorization", "Content-Type"], + }, + }; + case "tools.pageindex.run": + return { + resolved: true, + target: "service", + action: "tools_pageindex_run", + upstream: { + method: "POST", + url: `${toolsBridgeUrl()}/v1/pageindex/run`, + headersHint: ["Authorization", "Content-Type"], + }, + }; + case "tools.chandra.ocr": + return { + resolved: true, + target: "service", + action: "tools_chandra_ocr", + upstream: { + method: "POST", + url: `${toolsBridgeUrl()}/v1/chandra/ocr`, + headersHint: ["Authorization", "Content-Type"], + }, + }; default: return { resolved: false, reason: `Unknown intent: ${intent}` }; } diff --git a/services/smart-ide-tools-bridge/.env.example b/services/smart-ide-tools-bridge/.env.example new file mode 100644 index 0000000..90e4b93 --- /dev/null +++ b/services/smart-ide-tools-bridge/.env.example @@ -0,0 +1,14 @@ +# Copy to a local env file or export before npm start (do not commit secrets). +TOOLS_BRIDGE_HOST=127.0.0.1 +TOOLS_BRIDGE_PORT=37147 +TOOLS_BRIDGE_TOKEN= +TOOLS_BRIDGE_JOB_TIMEOUT_MS=3600000 + +# Path checks for POST /v1/pageindex/run and /v1/chandra/ocr +SMART_IDE_MONOREPO_ROOT= +# Optional comma-separated extra roots +# TOOLS_ALLOWED_PATH_PREFIXES= + +# Carbonyl open-plan response +CARBONYL_DOCKER_IMAGE=fathyb/carbonyl +CARBONYL_RUNNER=docker diff --git a/services/smart-ide-tools-bridge/.gitignore b/services/smart-ide-tools-bridge/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/services/smart-ide-tools-bridge/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/services/smart-ide-tools-bridge/README.md b/services/smart-ide-tools-bridge/README.md new file mode 100644 index 0000000..680615a --- /dev/null +++ b/services/smart-ide-tools-bridge/README.md @@ -0,0 +1,25 @@ +# smart-ide-tools-bridge + +HTTP local (**Bearer**) : **registre** des URLs des micro-services + API pour outils basés sur sous-modules (**Carbonyl**, **PageIndex**, **Chandra**). + +Voir [docs/repo/service-smart-ide-tools-bridge.md](../../docs/repo/service-smart-ide-tools-bridge.md) et [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md). + +Configuration agrégée : [config/services.local.env.example](../../config/services.local.env.example). + +## Run + +```bash +cd services/smart-ide-tools-bridge +npm install +npm run build +export TOOLS_BRIDGE_TOKEN='…' +export SMART_IDE_MONOREPO_ROOT="$(cd ../.. && pwd)" +npm start +``` + +## Prérequis jobs + +- **PageIndex** : Python / venv dans `services/pageindex/upstream` (voir `services/pageindex/README.md`). +- **Chandra** : install HF ou vLLM dans `services/chandra` (voir `services/chandra/README.md`). + +Les chemins passés aux POST doivent résoudre sous **`SMART_IDE_MONOREPO_ROOT`** (ou préfixes **`TOOLS_ALLOWED_PATH_PREFIXES`**). diff --git a/services/smart-ide-tools-bridge/package-lock.json b/services/smart-ide-tools-bridge/package-lock.json new file mode 100644 index 0000000..e88e506 --- /dev/null +++ b/services/smart-ide-tools-bridge/package-lock.json @@ -0,0 +1,51 @@ +{ + "name": "@4nk/smart-ide-tools-bridge", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@4nk/smart-ide-tools-bridge", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^20.11.0", + "typescript": "^5.3.3" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "20.19.39", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.39.tgz", + "integrity": "sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/services/smart-ide-tools-bridge/package.json b/services/smart-ide-tools-bridge/package.json new file mode 100644 index 0000000..58a82fb --- /dev/null +++ b/services/smart-ide-tools-bridge/package.json @@ -0,0 +1,20 @@ +{ + "name": "@4nk/smart-ide-tools-bridge", + "version": "0.1.0", + "private": true, + "description": "HTTP bridge: IDE-facing API for Carbonyl, PageIndex, Chandra + service registry.", + "license": "MIT", + "type": "module", + "main": "dist/server.js", + "scripts": { + "build": "tsc -p ./", + "start": "node dist/server.js" + }, + "engines": { + "node": ">=20" + }, + "devDependencies": { + "@types/node": "^20.11.0", + "typescript": "^5.3.3" + } +} diff --git a/services/smart-ide-tools-bridge/src/auth.ts b/services/smart-ide-tools-bridge/src/auth.ts new file mode 100644 index 0000000..2b1c1ce --- /dev/null +++ b/services/smart-ide-tools-bridge/src/auth.ts @@ -0,0 +1,21 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; + +export const readExpectedToken = (): string => { + return process.env.TOOLS_BRIDGE_TOKEN?.trim() ?? ""; +}; + +export const requireBearer = ( + req: IncomingMessage, + res: ServerResponse, + expected: string, +): boolean => { + const h = req.headers.authorization ?? ""; + const match = /^Bearer\s+(.+)$/i.exec(h); + const got = match?.[1]?.trim() ?? ""; + if (got !== expected) { + res.writeHead(401, { "Content-Type": "application/json; charset=utf-8" }); + res.end(JSON.stringify({ error: "Unauthorized" })); + return false; + } + return true; +}; diff --git a/services/smart-ide-tools-bridge/src/httpUtil.ts b/services/smart-ide-tools-bridge/src/httpUtil.ts new file mode 100644 index 0000000..2ed0c50 --- /dev/null +++ b/services/smart-ide-tools-bridge/src/httpUtil.ts @@ -0,0 +1,13 @@ +import type { IncomingMessage } from "node:http"; + +export const readJsonBody = async (req: IncomingMessage): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + const raw = Buffer.concat(chunks).toString("utf-8").trim(); + if (raw.length === 0) { + return {}; + } + return JSON.parse(raw) as unknown; +}; diff --git a/services/smart-ide-tools-bridge/src/paths.ts b/services/smart-ide-tools-bridge/src/paths.ts new file mode 100644 index 0000000..57170c8 --- /dev/null +++ b/services/smart-ide-tools-bridge/src/paths.ts @@ -0,0 +1,49 @@ +import path from "node:path"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; + +export const defaultMonorepoRoot = (): string => { + const fromEnv = process.env.SMART_IDE_MONOREPO_ROOT?.trim(); + if (fromEnv) { + return path.resolve(fromEnv); + } + const here = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(here, "..", "..", ".."); +}; + +const extraPrefixes = (): string[] => { + const raw = process.env.TOOLS_ALLOWED_PATH_PREFIXES?.trim(); + if (!raw) { + return []; + } + return raw + .split(",") + .map((s) => path.resolve(s.trim())) + .filter((s) => s.length > 0); +}; + +export const isPathAllowed = (absPath: string, monorepoRoot: string): boolean => { + const norm = path.resolve(absPath); + const roots = [path.resolve(monorepoRoot), ...extraPrefixes()]; + return roots.some((r) => norm === r || norm.startsWith(r + path.sep)); +}; + +export const assertAllowedFile = (p: string, monorepoRoot: string): string => { + const abs = path.resolve(p); + if (!isPathAllowed(abs, monorepoRoot)) { + throw new Error(`Path not allowed under SMART_IDE_MONOREPO_ROOT: ${abs}`); + } + if (!fs.existsSync(abs)) { + throw new Error(`Path does not exist: ${abs}`); + } + return abs; +}; + +/** Output directory (may not exist yet); must resolve under an allowed root. */ +export const assertAllowedDirPath = (p: string, monorepoRoot: string): string => { + const abs = path.resolve(p); + if (!isPathAllowed(abs, monorepoRoot)) { + throw new Error(`Path not allowed under SMART_IDE_MONOREPO_ROOT: ${abs}`); + } + return abs; +}; diff --git a/services/smart-ide-tools-bridge/src/server.ts b/services/smart-ide-tools-bridge/src/server.ts new file mode 100644 index 0000000..22253d0 --- /dev/null +++ b/services/smart-ide-tools-bridge/src/server.ts @@ -0,0 +1,261 @@ +import * as http from "node:http"; +import path from "node:path"; +import fs from "node:fs"; +import { readExpectedToken, requireBearer } from "./auth.js"; +import { readJsonBody } from "./httpUtil.js"; +import { + assertAllowedDirPath, + assertAllowedFile, + defaultMonorepoRoot, +} from "./paths.js"; +import { runProcess } from "./spawnJob.js"; + +const HOST = process.env.TOOLS_BRIDGE_HOST ?? "127.0.0.1"; +const PORT = Number(process.env.TOOLS_BRIDGE_PORT ?? "37147"); +const JOB_TIMEOUT_MS = Number( + process.env.TOOLS_BRIDGE_JOB_TIMEOUT_MS ?? "3600000", +); + +const json = (res: http.ServerResponse, status: number, body: unknown): void => { + res.writeHead(status, { "Content-Type": "application/json; charset=utf-8" }); + res.end(JSON.stringify(body)); +}; + +const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); + +const safeHttpUrl = (raw: string): string => { + const u = new URL(raw); + if (u.protocol !== "http:" && u.protocol !== "https:") { + throw new Error("Only http and https URLs are allowed"); + } + return u.toString(); +}; + +const serviceRegistry = (): Record => ({ + monorepoRoot: defaultMonorepoRoot(), + services: { + tools_bridge: { + url: `http://${HOST}:${PORT}`, + tokenEnv: "TOOLS_BRIDGE_TOKEN", + }, + repos_devtools: { + url: (process.env.REPOS_DEVTOOLS_URL ?? `http://127.0.0.1:${process.env.REPOS_DEVTOOLS_PORT ?? "37140"}`).replace( + /\/+$/, + "", + ), + tokenEnv: "REPOS_DEVTOOLS_TOKEN", + }, + langextract: { + url: `http://${process.env.LANGEXTRACT_API_HOST ?? "127.0.0.1"}:${process.env.LANGEXTRACT_API_PORT ?? "37141"}`, + tokenEnv: "LANGEXTRACT_SERVICE_TOKEN", + }, + claw_proxy: { + url: `http://${process.env.CLAW_PROXY_HOST ?? "127.0.0.1"}:${process.env.CLAW_PROXY_PORT ?? "37142"}`, + tokenEnv: "CLAW_PROXY_TOKEN", + }, + regex_search: { + url: `http://${process.env.REGEX_SEARCH_HOST ?? "127.0.0.1"}:${process.env.REGEX_SEARCH_PORT ?? "37143"}`, + tokenEnv: "REGEX_SEARCH_TOKEN", + }, + local_office: { + url: (process.env.LOCAL_OFFICE_URL ?? "http://127.0.0.1:8000").replace(/\/+$/, ""), + tokenEnv: "LOCAL_OFFICE_API_KEY", + header: "X-API-Key", + }, + ia_dev_gateway: { + url: `http://${process.env.IA_DEV_GATEWAY_HOST ?? "127.0.0.1"}:${process.env.IA_DEV_GATEWAY_PORT ?? "37144"}`, + tokenEnv: "IA_DEV_GATEWAY_TOKEN", + }, + orchestrator: { + url: `http://${process.env.ORCHESTRATOR_HOST ?? "127.0.0.1"}:${process.env.ORCHESTRATOR_PORT ?? "37145"}`, + tokenEnv: "ORCHESTRATOR_TOKEN", + }, + anythingllm_devtools: { + url: `http://${process.env.ANYTHINGLLM_DEVTOOLS_HOST ?? "127.0.0.1"}:${process.env.ANYTHINGLLM_DEVTOOLS_PORT ?? "37146"}`, + tokenEnv: "ANYTHINGLLM_DEVTOOLS_TOKEN", + }, + }, + submodule_tools: { + carbonyl: { + openPlanPath: "/v1/carbonyl/open-plan", + upstreamSubmodule: "services/carbonyl/upstream", + }, + pageindex: { + runPath: "/v1/pageindex/run", + upstreamSubmodule: "services/pageindex/upstream", + }, + chandra: { + ocrPath: "/v1/chandra/ocr", + upstreamSubmodule: "services/chandra/upstream", + }, + }, +}); + +const main = (): void => { + const token = readExpectedToken(); + if (token.length === 0) { + console.error("smart-ide-tools-bridge: set TOOLS_BRIDGE_TOKEN (non-empty secret)."); + process.exit(1); + } + + const monorepoRoot = defaultMonorepoRoot(); + + const server = http.createServer((req, res) => { + void (async () => { + try { + const url = new URL(req.url ?? "/", `http://${HOST}`); + const p = url.pathname; + + if (req.method === "GET" && (p === "/health" || p === "/health/")) { + json(res, 200, { status: "ok", service: "smart-ide-tools-bridge" }); + return; + } + + if (req.method === "GET" && p === "/v1/registry") { + if (!requireBearer(req, res, token)) { + return; + } + json(res, 200, serviceRegistry()); + return; + } + + if (req.method === "POST" && p === "/v1/carbonyl/open-plan") { + if (!requireBearer(req, res, token)) { + return; + } + const body = await readJsonBody(req); + if (!isRecord(body) || typeof body.url !== "string") { + json(res, 422, { error: "Missing url (string)" }); + return; + } + const targetUrl = safeHttpUrl(body.url); + const image = process.env.CARBONYL_DOCKER_IMAGE ?? "fathyb/carbonyl"; + const runner = (process.env.CARBONYL_RUNNER ?? "docker").toLowerCase(); + if (runner === "native") { + json(res, 200, { + runner: "native", + argv: ["carbonyl", targetUrl], + hint: "Run in a terminal; requires carbonyl in PATH.", + }); + return; + } + json(res, 200, { + runner: "docker", + argv: ["docker", "run", "--rm", "-ti", image, targetUrl], + hint: "Interactive TTY (-ti) required; run from a terminal session.", + }); + return; + } + + if (req.method === "POST" && p === "/v1/pageindex/run") { + if (!requireBearer(req, res, token)) { + return; + } + const body = await readJsonBody(req); + if (!isRecord(body) || typeof body.inputPath !== "string") { + json(res, 422, { error: "Missing inputPath (string)" }); + return; + } + const mode = + body.mode === "md" ? "md" : body.mode === "pdf" ? "pdf" : null; + if (!mode) { + json(res, 422, { error: "mode must be pdf or md" }); + return; + } + const inputAbs = assertAllowedFile(body.inputPath, monorepoRoot); + const pageindexDir = path.join(monorepoRoot, "services", "pageindex"); + const script = path.join(pageindexDir, "run-pageindex.sh"); + if (!fs.existsSync(script)) { + json(res, 500, { error: "run-pageindex.sh missing" }); + return; + } + const flag = mode === "pdf" ? "--pdf_path" : "--md_path"; + const { code, stdout, stderr } = await runProcess(script, [flag, inputAbs], { + cwd: pageindexDir, + timeoutMs: JOB_TIMEOUT_MS, + }); + const base = path.basename(inputAbs, path.extname(inputAbs)); + const resultPath = path.join( + monorepoRoot, + "services", + "pageindex", + "upstream", + "results", + `${base}_structure.json`, + ); + json(res, code === 0 ? 200 : 500, { + ok: code === 0, + exitCode: code, + stdoutTail: stdout.slice(-8000), + stderrTail: stderr.slice(-8000), + resultPath: fs.existsSync(resultPath) ? resultPath : null, + }); + return; + } + + if (req.method === "POST" && p === "/v1/chandra/ocr") { + if (!requireBearer(req, res, token)) { + return; + } + const body = await readJsonBody(req); + if ( + !isRecord(body) || + typeof body.inputPath !== "string" || + typeof body.outputPath !== "string" + ) { + json(res, 422, { + error: "Missing inputPath (string) or outputPath (string)", + }); + return; + } + const inputAbs = assertAllowedFile(body.inputPath, monorepoRoot); + const outputAbs = assertAllowedDirPath(body.outputPath, monorepoRoot); + fs.mkdirSync(outputAbs, { recursive: true }); + const method = + body.method === "vllm" ? "vllm" : body.method === "hf" ? "hf" : "hf"; + const chandraDir = path.join(monorepoRoot, "services", "chandra"); + const runHf = path.join(chandraDir, "run-chandra-hf.sh"); + const runAny = path.join(chandraDir, "run-chandra.sh"); + if (!fs.existsSync(runAny)) { + json(res, 500, { error: "Chandra run script missing" }); + return; + } + const script = + method === "hf" && fs.existsSync(runHf) + ? runHf + : runAny; + const args = + method === "hf" && script === runHf + ? [inputAbs, outputAbs] + : method === "hf" + ? [inputAbs, outputAbs, "--method", "hf"] + : [inputAbs, outputAbs, "--method", "vllm"]; + const { code, stdout, stderr } = await runProcess(script, args, { + cwd: chandraDir, + timeoutMs: JOB_TIMEOUT_MS, + }); + json(res, code === 0 ? 200 : 500, { + ok: code === 0, + exitCode: code, + outputDir: outputAbs, + stdoutTail: stdout.slice(-8000), + stderrTail: stderr.slice(-8000), + }); + return; + } + + json(res, 404, { error: "Not found" }); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + json(res, 400, { error: msg }); + } + })(); + }); + + server.listen(PORT, HOST, () => { + console.error(`smart-ide-tools-bridge listening on http://${HOST}:${PORT}`); + }); +}; + +main(); diff --git a/services/smart-ide-tools-bridge/src/spawnJob.ts b/services/smart-ide-tools-bridge/src/spawnJob.ts new file mode 100644 index 0000000..c9ce536 --- /dev/null +++ b/services/smart-ide-tools-bridge/src/spawnJob.ts @@ -0,0 +1,35 @@ +import { spawn } from "node:child_process"; + +export const runProcess = ( + command: string, + args: string[], + options: { cwd: string; timeoutMs: number }, +): Promise<{ code: number | null; stdout: string; stderr: string }> => { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const t = setTimeout(() => { + child.kill("SIGTERM"); + reject(new Error(`Process timeout after ${options.timeoutMs}ms`)); + }, options.timeoutMs); + child.stdout?.on("data", (d: Buffer) => { + stdout += d.toString("utf-8"); + }); + child.stderr?.on("data", (d: Buffer) => { + stderr += d.toString("utf-8"); + }); + child.on("close", (code) => { + clearTimeout(t); + resolve({ code, stdout, stderr }); + }); + child.on("error", (e) => { + clearTimeout(t); + reject(e); + }); + }); +}; diff --git a/services/smart-ide-tools-bridge/tsconfig.json b/services/smart-ide-tools-bridge/tsconfig.json new file mode 100644 index 0000000..ae73f32 --- /dev/null +++ b/services/smart-ide-tools-bridge/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "skipLibCheck": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "declaration": false + }, + "include": ["src/**/*.ts"] +}