Add smart-ide-tools-bridge API for submodule tools + central local env
- New service: tools bridge (port 37147) registry + Carbonyl/PageIndex/Chandra POST jobs - config/services.local.env.example and gitignore for services.local.env - .env.example for repos-devtools, regex-search, ia-dev-gateway, orchestrator, claw proxy, langextract - Orchestrator intents: tools.registry, tools.carbonyl.plan, tools.pageindex.run, tools.chandra.ocr - Docs: API + repo service fiche, architecture index; do not commit dist/
This commit is contained in:
parent
d6a61e7cbe
commit
14c974f54c
3
.gitignore
vendored
3
.gitignore
vendored
@ -9,6 +9,9 @@ services/docv/target/
|
|||||||
# Surcharges locales pull-sync (cron)
|
# Surcharges locales pull-sync (cron)
|
||||||
cron/config.local.env
|
cron/config.local.env
|
||||||
|
|
||||||
|
# Configuration locale agrégée des services (copie de config/services.local.env.example)
|
||||||
|
config/services.local.env
|
||||||
|
|
||||||
# logs/ : le répertoire reste versionné (README.md, logs/.gitignore) ; aucun fichier journal versionné
|
# logs/ : le répertoire reste versionné (README.md, logs/.gitignore) ; aucun fichier journal versionné
|
||||||
logs/**/*.log
|
logs/**/*.log
|
||||||
|
|
||||||
|
|||||||
3
config/README.md
Normal file
3
config/README.md
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
# Configuration locale
|
||||||
|
|
||||||
|
- **`services.local.env.example`** : variables pour l’IDE et tous les services HTTP ; copier vers **`services.local.env`** (non versionné, voir racine **`.gitignore`**).
|
||||||
65
config/services.local.env.example
Normal file
65
config/services.local.env.example
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
# Local IDE / agent configuration for all smart_ide HTTP services and the tools bridge.
|
||||||
|
# Copy to config/services.local.env and fill secrets. Do not commit services.local.env.
|
||||||
|
#
|
||||||
|
# Load before starting processes, e.g.: set -a && source config/services.local.env && set +a
|
||||||
|
|
||||||
|
# Monorepo root (path validation for tools-bridge jobs)
|
||||||
|
SMART_IDE_MONOREPO_ROOT=/absolute/path/to/smart_ide
|
||||||
|
|
||||||
|
# Optional: extra allowed path prefixes (comma-separated) for tools-bridge file jobs
|
||||||
|
# TOOLS_ALLOWED_PATH_PREFIXES=/data/clones,/home/user/projects
|
||||||
|
|
||||||
|
# --- smart-ide-tools-bridge (Carbonyl / PageIndex / Chandra + registry) ---
|
||||||
|
TOOLS_BRIDGE_HOST=127.0.0.1
|
||||||
|
TOOLS_BRIDGE_PORT=37147
|
||||||
|
TOOLS_BRIDGE_TOKEN=
|
||||||
|
TOOLS_BRIDGE_JOB_TIMEOUT_MS=3600000
|
||||||
|
|
||||||
|
# --- Core HTTP micro-services ---
|
||||||
|
REPOS_DEVTOOLS_HOST=127.0.0.1
|
||||||
|
REPOS_DEVTOOLS_PORT=37140
|
||||||
|
REPOS_DEVTOOLS_TOKEN=
|
||||||
|
REPOS_DEVTOOLS_ROOT=
|
||||||
|
|
||||||
|
LANGEXTRACT_API_HOST=127.0.0.1
|
||||||
|
LANGEXTRACT_API_PORT=37141
|
||||||
|
LANGEXTRACT_SERVICE_TOKEN=
|
||||||
|
|
||||||
|
CLAW_PROXY_HOST=127.0.0.1
|
||||||
|
CLAW_PROXY_PORT=37142
|
||||||
|
CLAW_PROXY_TOKEN=
|
||||||
|
CLAW_UPSTREAM_URL=
|
||||||
|
|
||||||
|
REGEX_SEARCH_HOST=127.0.0.1
|
||||||
|
REGEX_SEARCH_PORT=37143
|
||||||
|
REGEX_SEARCH_TOKEN=
|
||||||
|
REGEX_SEARCH_ROOT=
|
||||||
|
|
||||||
|
LOCAL_OFFICE_URL=http://127.0.0.1:8000
|
||||||
|
LOCAL_OFFICE_API_KEY=
|
||||||
|
|
||||||
|
IA_DEV_GATEWAY_HOST=127.0.0.1
|
||||||
|
IA_DEV_GATEWAY_PORT=37144
|
||||||
|
IA_DEV_GATEWAY_TOKEN=
|
||||||
|
|
||||||
|
ORCHESTRATOR_HOST=127.0.0.1
|
||||||
|
ORCHESTRATOR_PORT=37145
|
||||||
|
ORCHESTRATOR_TOKEN=
|
||||||
|
|
||||||
|
ANYTHINGLLM_DEVTOOLS_HOST=127.0.0.1
|
||||||
|
ANYTHINGLLM_DEVTOOLS_PORT=37146
|
||||||
|
ANYTHINGLLM_DEVTOOLS_TOKEN=
|
||||||
|
ANYTHINGLLM_BASE_URL=
|
||||||
|
ANYTHINGLLM_API_KEY=
|
||||||
|
REPOS_DEVTOOLS_URL=http://127.0.0.1:37140
|
||||||
|
|
||||||
|
# Orchestrator → tools bridge (for intent resolution hints)
|
||||||
|
TOOLS_BRIDGE_URL=http://127.0.0.1:37147
|
||||||
|
|
||||||
|
# Ollama / AnythingLLM (orchestrator)
|
||||||
|
OLLAMA_URL=http://127.0.0.1:11434
|
||||||
|
ANYTHINGLLM_BASE_URL=
|
||||||
|
|
||||||
|
# Carbonyl (docker / native — used by bridge open-plan response)
|
||||||
|
CARBONYL_DOCKER_IMAGE=fathyb/carbonyl
|
||||||
|
CARBONYL_RUNNER=docker
|
||||||
@ -12,6 +12,7 @@ Documentation des **API HTTP** exposées par les services sous [`services/`](../
|
|||||||
| **ia-dev-gateway** | Bearer | `37144` (spécification) | [ia-dev-gateway.md](./ia-dev-gateway.md) |
|
| **ia-dev-gateway** | Bearer | `37144` (spécification) | [ia-dev-gateway.md](./ia-dev-gateway.md) |
|
||||||
| **smart_ide-orchestrator** | Bearer (spécification) | `37145` (spécification) | [orchestrator.md](./orchestrator.md) |
|
| **smart_ide-orchestrator** | Bearer (spécification) | `37145` (spécification) | [orchestrator.md](./orchestrator.md) |
|
||||||
| **anythingllm-devtools** | Bearer | `37146` | [anythingllm-devtools-api.md](./anythingllm-devtools-api.md) |
|
| **anythingllm-devtools** | Bearer | `37146` | [anythingllm-devtools-api.md](./anythingllm-devtools-api.md) |
|
||||||
|
| **smart-ide-tools-bridge** | Bearer (sauf `/health`) | `37147` | [smart-ide-tools-bridge-api.md](./smart-ide-tools-bridge-api.md) |
|
||||||
| **docv** (externe) | selon dépôt Enso | selon déploiement | [docv.md](./docv.md) |
|
| **docv** (externe) | selon dépôt Enso | selon déploiement | [docv.md](./docv.md) |
|
||||||
|
|
||||||
**OpenAPI** : FastAPI expose une spec interactive pour **langextract-api** (`/docs`) et **local-office** (`/docs`) une fois le service démarré.
|
**OpenAPI** : FastAPI expose une spec interactive pour **langextract-api** (`/docs`) et **local-office** (`/docs`) une fois le service démarré.
|
||||||
|
|||||||
@ -14,7 +14,7 @@ Résout une intention sans nécessairement l’exécuter (si `dryRun: true`).
|
|||||||
|
|
||||||
| Champ | Obligatoire | Description |
|
| Champ | Obligatoire | Description |
|
||||||
|-------|-------------|-------------|
|
|-------|-------------|-------------|
|
||||||
| `intent` | oui | Identifiant stable (`code.complete`, `rag.query`, `agent.run`, …) |
|
| `intent` | oui | Identifiant stable (`code.complete`, `rag.query`, `agent.run`, `tools.registry`, `tools.pageindex.run`, …) |
|
||||||
| `context` | non | Objet libre (fichiers ouverts, sélection, etc.) |
|
| `context` | non | Objet libre (fichiers ouverts, sélection, etc.) |
|
||||||
| `projectId` | non | Projet `ia_dev` / workspace |
|
| `projectId` | non | Projet `ia_dev` / workspace |
|
||||||
| `env` | non | `test` \| `pprod` \| `prod` |
|
| `env` | non | `test` \| `pprod` \| `prod` |
|
||||||
|
|||||||
42
docs/API/smart-ide-tools-bridge-api.md
Normal file
42
docs/API/smart-ide-tools-bridge-api.md
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
# API smart-ide-tools-bridge
|
||||||
|
|
||||||
|
Écoute **`127.0.0.1`** par défaut, port **`37147`** (`TOOLS_BRIDGE_PORT`). Auth : **`Authorization: Bearer <TOOLS_BRIDGE_TOKEN>`** (obligatoire sauf **`GET /health`**).
|
||||||
|
|
||||||
|
## Endpoints
|
||||||
|
|
||||||
|
| Méthode | Chemin | Rôle |
|
||||||
|
|---------|--------|------|
|
||||||
|
| GET | `/health` | Santé (sans Bearer). |
|
||||||
|
| GET | `/v1/registry` | JSON : URLs des services HTTP du monorepo + métadonnées des outils sous-module (chemins relatifs). |
|
||||||
|
| POST | `/v1/carbonyl/open-plan` | Corps `{"url":"https://…"}` → plan d’exécution terminal (`docker` ou `native`). |
|
||||||
|
| POST | `/v1/pageindex/run` | Corps `{"mode":"pdf"\|"md","inputPath":"/abs/…"}` → sous-processus `run-pageindex.sh` ; réponse inclut `resultPath` si `…/upstream/results/<stem>_structure.json` existe. |
|
||||||
|
| POST | `/v1/chandra/ocr` | Corps `{"inputPath":"…","outputPath":"…","method":"hf"\|"vllm"}` (défaut `hf`) → scripts Chandra. |
|
||||||
|
|
||||||
|
## Corps JSON (exemples)
|
||||||
|
|
||||||
|
**Carbonyl**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "url": "https://example.com" }
|
||||||
|
```
|
||||||
|
|
||||||
|
**PageIndex**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "mode": "pdf", "inputPath": "/abs/smart_ide/docs/foo.pdf" }
|
||||||
|
```
|
||||||
|
|
||||||
|
**Chandra**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{ "inputPath": "/abs/in.pdf", "outputPath": "/abs/out_dir", "method": "hf" }
|
||||||
|
```
|
||||||
|
|
||||||
|
## Sécurité des chemins
|
||||||
|
|
||||||
|
`inputPath` / `outputPath` doivent résoudre sous **`SMART_IDE_MONOREPO_ROOT`** ou sous un préfixe listé dans **`TOOLS_ALLOWED_PATH_PREFIXES`** (séparateur virgule).
|
||||||
|
|
||||||
|
## Voir aussi
|
||||||
|
|
||||||
|
- [README.md](../README.md) (table des ports)
|
||||||
|
- [repo/service-smart-ide-tools-bridge.md](../repo/service-smart-ide-tools-bridge.md)
|
||||||
@ -2,6 +2,13 @@
|
|||||||
|
|
||||||
Index principal. Les **fonctionnalités** détaillées sont dans [`features/`](./features/). La **documentation opérationnelle** regroupée (anciens `README.md` à la racine et sous `cron/`, `projects/`, `services/`, etc.) est dans **[`repo/`](./repo/)**.
|
Index principal. Les **fonctionnalités** détaillées sont dans [`features/`](./features/). La **documentation opérationnelle** regroupée (anciens `README.md` à la racine et sous `cron/`, `projects/`, `services/`, etc.) est dans **[`repo/`](./repo/)**.
|
||||||
|
|
||||||
|
## Configuration locale agrégée
|
||||||
|
|
||||||
|
| Fichier | Rôle |
|
||||||
|
|---------|------|
|
||||||
|
| [config/services.local.env.example](../config/services.local.env.example) | Ports, jetons, `SMART_IDE_MONOREPO_ROOT` — copier vers `config/services.local.env` |
|
||||||
|
| [config/README.md](../config/README.md) | Rôle du répertoire `config/` |
|
||||||
|
|
||||||
## Documentation du dépôt (`repo/`)
|
## Documentation du dépôt (`repo/`)
|
||||||
|
|
||||||
Vue d’ensemble et index complet : **[repo/README.md](./repo/README.md)**. Règles/agents IDE : **[repo/smartide-config-directory.md](./repo/smartide-config-directory.md)**.
|
Vue d’ensemble et index complet : **[repo/README.md](./repo/README.md)**. Règles/agents IDE : **[repo/smartide-config-directory.md](./repo/smartide-config-directory.md)**.
|
||||||
@ -25,6 +32,7 @@ Vue d’ensemble et index complet : **[repo/README.md](./repo/README.md)**. Règ
|
|||||||
| [repo/service-anythingllm-devtools.md](./repo/service-anythingllm-devtools.md) | Service HTTP AnythingLLM + devtools |
|
| [repo/service-anythingllm-devtools.md](./repo/service-anythingllm-devtools.md) | Service HTTP AnythingLLM + devtools |
|
||||||
| [repo/service-carbonyl.md](./repo/service-carbonyl.md) | Carbonyl (navigateur terminal), prévisualisation test |
|
| [repo/service-carbonyl.md](./repo/service-carbonyl.md) | Carbonyl (navigateur terminal), prévisualisation test |
|
||||||
| [repo/service-pageindex.md](./repo/service-pageindex.md) | PageIndex (index vectorless, définition sémantique documents) |
|
| [repo/service-pageindex.md](./repo/service-pageindex.md) | PageIndex (index vectorless, définition sémantique documents) |
|
||||||
|
| [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md) | Pont HTTP IDE + sous-modules CLI |
|
||||||
| [repo/service-chandra.md](./repo/service-chandra.md) | Chandra OCR (PDF / images structurés) |
|
| [repo/service-chandra.md](./repo/service-chandra.md) | Chandra OCR (PDF / images structurés) |
|
||||||
| [repo/extension-anythingllm-workspaces.md](./repo/extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; voir anythingllm-devtools) |
|
| [repo/extension-anythingllm-workspaces.md](./repo/extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; voir anythingllm-devtools) |
|
||||||
|
|
||||||
|
|||||||
@ -37,6 +37,7 @@ La référence OpenAPI détaillée : [API/orchestrator.md](../API/orchestrator.m
|
|||||||
| `extract.entities` | langextract-api |
|
| `extract.entities` | langextract-api |
|
||||||
| `doc.office.upload` | local-office (`X-API-Key`) |
|
| `doc.office.upload` | local-office (`X-API-Key`) |
|
||||||
| `agent.run`, `deploy.trigger` | ia-dev-gateway |
|
| `agent.run`, `deploy.trigger` | ia-dev-gateway |
|
||||||
|
| `tools.registry`, `tools.carbonyl.plan`, `tools.pageindex.run`, `tools.chandra.ocr` | smart-ide-tools-bridge (`TOOLS_BRIDGE_URL`, jeton `TOOLS_BRIDGE_TOKEN`) |
|
||||||
|
|
||||||
## Variables d’environnement (cible)
|
## Variables d’environnement (cible)
|
||||||
|
|
||||||
@ -49,6 +50,7 @@ La référence OpenAPI détaillée : [API/orchestrator.md](../API/orchestrator.m
|
|||||||
| `ANYTHINGLLM_API_KEY` | Clé API |
|
| `ANYTHINGLLM_API_KEY` | Clé API |
|
||||||
| `REPOS_DEVTOOLS_URL`, `REPOS_DEVTOOLS_TOKEN` | … |
|
| `REPOS_DEVTOOLS_URL`, `REPOS_DEVTOOLS_TOKEN` | … |
|
||||||
| `IA_DEV_GATEWAY_URL`, `IA_DEV_GATEWAY_TOKEN` | … |
|
| `IA_DEV_GATEWAY_URL`, `IA_DEV_GATEWAY_TOKEN` | … |
|
||||||
|
| `TOOLS_BRIDGE_URL` | Base URL du pont IDE (défaut `http://127.0.0.1:37147`) |
|
||||||
|
|
||||||
Les valeurs diffèrent par **environnement** (test / pprod / prod) — voir [platform-target.md](../platform-target.md).
|
Les valeurs diffèrent par **environnement** (test / pprod / prod) — voir [platform-target.md](../platform-target.md).
|
||||||
|
|
||||||
|
|||||||
@ -40,6 +40,7 @@ Toute la documentation **opérationnelle** qui vivait auparavant sous des `READM
|
|||||||
| [service-carbonyl.md](./service-carbonyl.md) | Carbonyl (navigateur terminal), sous-module amont |
|
| [service-carbonyl.md](./service-carbonyl.md) | Carbonyl (navigateur terminal), sous-module amont |
|
||||||
| [service-pageindex.md](./service-pageindex.md) | PageIndex (index sémantique vectorless), sous-module amont |
|
| [service-pageindex.md](./service-pageindex.md) | PageIndex (index sémantique vectorless), sous-module amont |
|
||||||
| [service-chandra.md](./service-chandra.md) | Chandra OCR, sous-module amont |
|
| [service-chandra.md](./service-chandra.md) | Chandra OCR, sous-module amont |
|
||||||
|
| [service-smart-ide-tools-bridge.md](./service-smart-ide-tools-bridge.md) | Pont HTTP IDE + outils sous-modules |
|
||||||
| [extension-anythingllm-workspaces.md](./extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; anythingllm-devtools) |
|
| [extension-anythingllm-workspaces.md](./extension-anythingllm-workspaces.md) | Extension AnythingLLM IDE (supprimée ; anythingllm-devtools) |
|
||||||
|
|
||||||
Les **spécifications** détaillées (contrats HTTP, sécurité, orchestration) restent dans [../API/README.md](../API/README.md) et [../features/](../features/).
|
Les **spécifications** détaillées (contrats HTTP, sécurité, orchestration) restent dans [../API/README.md](../API/README.md) et [../features/](../features/).
|
||||||
|
|||||||
@ -6,8 +6,9 @@ Navigateur **terminal** basé sur Chromium — amont **[fathyb/carbonyl](https:/
|
|||||||
|
|
||||||
- **Pilotage / visualisation** d’URLs dans un terminal (SSH sans GUI, session locale).
|
- **Pilotage / visualisation** d’URLs dans un terminal (SSH sans GUI, session locale).
|
||||||
- **Prévisualisation** des applications déployées en **test** : URL optionnelle dans **`projects/<id>/conf.json`** → **`smart_ide.preview_urls.test`**, ouverte par **`scripts/open-carbonyl-preview-test.sh`**.
|
- **Prévisualisation** des applications déployées en **test** : URL optionnelle dans **`projects/<id>/conf.json`** → **`smart_ide.preview_urls.test`**, ouverte par **`scripts/open-carbonyl-preview-test.sh`**.
|
||||||
|
- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/carbonyl/open-plan` (retourne la commande à lancer en terminal) — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
Ce n’est **pas** un service HTTP : pas de port d’écoute dans smart_ide. L’exécution est un **processus interactif** (Docker ou binaire `carbonyl`).
|
Carbonyl lui-même reste un **processus interactif** (Docker ou binaire `carbonyl`), pas un listener HTTP dédié.
|
||||||
|
|
||||||
## Exploitation
|
## Exploitation
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,8 @@ OCR et extraction **structurée** (PDF / images → Markdown, HTML, JSON avec mi
|
|||||||
## Rôle dans smart_ide
|
## Rôle dans smart_ide
|
||||||
|
|
||||||
- **Numérisation** de documents complexes (tableaux, formulaires, manuscrits, math).
|
- **Numérisation** de documents complexes (tableaux, formulaires, manuscrits, math).
|
||||||
- **Pas de listener HTTP** dans ce dépôt : CLI **`chandra`**, lancée par **`services/chandra/run-chandra.sh`** après installation dans **`upstream/`** (`uv sync` ou équivalent).
|
- **CLI** : **`services/chandra/run-chandra.sh`** / **`run-chandra-hf.sh`** après installation dans **`upstream/`**.
|
||||||
|
- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/chandra/ocr` — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
## Licences
|
## Licences
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,8 @@ Indexation **sémantique structurée** de documents longs (PDF, Markdown) via le
|
|||||||
## Rôle dans smart_ide
|
## Rôle dans smart_ide
|
||||||
|
|
||||||
- **Pilotage de la définition sémantique** des documents : produire ou exploiter un **index arborescent** traçable (titres, pages, nœuds), distinct du RAG par embeddings **AnythingLLM**.
|
- **Pilotage de la définition sémantique** des documents : produire ou exploiter un **index arborescent** traçable (titres, pages, nœuds), distinct du RAG par embeddings **AnythingLLM**.
|
||||||
- **Pas de service HTTP** dans ce dépôt : exécution **CLI** Python sous **`services/pageindex/upstream/`**, lancée via **`services/pageindex/run-pageindex.sh`**.
|
- **CLI** : **`services/pageindex/run-pageindex.sh`** (Python sous **`upstream/`**).
|
||||||
|
- **API pour l’IDE** : **`services/smart-ide-tools-bridge/`** — `POST /v1/pageindex/run` — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
## Exploitation
|
## Exploitation
|
||||||
|
|
||||||
|
|||||||
18
docs/repo/service-smart-ide-tools-bridge.md
Normal file
18
docs/repo/service-smart-ide-tools-bridge.md
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
# Service smart-ide-tools-bridge (`services/smart-ide-tools-bridge/`)
|
||||||
|
|
||||||
|
Pont HTTP pour l’**IDE** et les agents : exposition d’une **API** au-dessus des outils **CLI** (sous-modules **Carbonyl**, **PageIndex**, **Chandra**) et **registre** des autres services locaux (URLs + noms de variables de jeton, sans secrets).
|
||||||
|
|
||||||
|
## Configuration locale
|
||||||
|
|
||||||
|
- Fichier agrégé : **[`config/services.local.env.example`](../../config/services.local.env.example)** → copier en **`config/services.local.env`** (gitignoré).
|
||||||
|
- Service : **`services/smart-ide-tools-bridge/.env.example`** (`TOOLS_BRIDGE_TOKEN`, `SMART_IDE_MONOREPO_ROOT`, `TOOLS_ALLOWED_PATH_PREFIXES`, délais).
|
||||||
|
|
||||||
|
## Exploitation
|
||||||
|
|
||||||
|
Voir **[`services/smart-ide-tools-bridge/README.md`](../../services/smart-ide-tools-bridge/README.md)** et **[API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md)**.
|
||||||
|
|
||||||
|
**Orchestrateur** : intents `tools.registry`, `tools.carbonyl.plan`, `tools.pageindex.run`, `tools.chandra.ocr` — variable **`TOOLS_BRIDGE_URL`**.
|
||||||
|
|
||||||
|
## Voir aussi
|
||||||
|
|
||||||
|
- [service-carbonyl.md](./service-carbonyl.md), [service-pageindex.md](./service-pageindex.md), [service-chandra.md](./service-chandra.md)
|
||||||
@ -38,7 +38,11 @@ Services d’appoint sur **`127.0.0.1`** (souvent auth **Bearer**) : Git devtool
|
|||||||
|
|
||||||
**PageIndex** (`services/pageindex/`) n’est pas un listener HTTP : outil Python (sous-module [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex)) pour produire un **index arborescent** sémantique sur PDF / Markdown, en complément du RAG **AnythingLLM** — [repo/service-pageindex.md](./repo/service-pageindex.md).
|
**PageIndex** (`services/pageindex/`) n’est pas un listener HTTP : outil Python (sous-module [VectifyAI/PageIndex](https://github.com/VectifyAI/PageIndex)) pour produire un **index arborescent** sémantique sur PDF / Markdown, en complément du RAG **AnythingLLM** — [repo/service-pageindex.md](./repo/service-pageindex.md).
|
||||||
|
|
||||||
**Chandra OCR** (`services/chandra/`) n’est pas un listener HTTP : CLI (sous-module [datalab-to/chandra](https://github.com/datalab-to/chandra)) pour **OCR** PDF / images vers Markdown, HTML, JSON avec layout — [repo/service-chandra.md](./repo/service-chandra.md).
|
**Chandra OCR** (`services/chandra/`) n’est pas un listener HTTP : CLI (sous-module [datalab-to/chandra](https://github.com/datalab-to/chandra)) pour **OCR** PDF / images vers Markdown, HTML, JSON avec layout — [repo/service-chandra.md](./repo/service-chandra.md). L’**IDE** peut piloter Chandra via **`services/smart-ide-tools-bridge/`** — [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md).
|
||||||
|
|
||||||
|
**smart-ide-tools-bridge** (`services/smart-ide-tools-bridge/`) : **API** locale (registre + Carbonyl / PageIndex / Chandra) — [API/smart-ide-tools-bridge-api.md](../API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
|
**Configuration locale** : [config/services.local.env.example](../config/services.local.env.example).
|
||||||
|
|
||||||
## Documentation liée
|
## Documentation liée
|
||||||
|
|
||||||
|
|||||||
@ -44,6 +44,8 @@ Conséquences :
|
|||||||
| `ia_dev/` | Agents, déploiements — exécution sous policy ; `ia_dev/projects/<id>` peut pointer vers `../../projects/<id>` (lien) ; voir [ia_dev-module.md](./ia_dev-module.md) |
|
| `ia_dev/` | Agents, déploiements — exécution sous policy ; `ia_dev/projects/<id>` peut pointer vers `../../projects/<id>` (lien) ; voir [ia_dev-module.md](./ia_dev-module.md) |
|
||||||
| `services/ia-dev-gateway/` | Gateway HTTP (stub runner) : registre agents `.md`, runs, SSE — [features/ia-dev-service.md](./features/ia-dev-service.md) |
|
| `services/ia-dev-gateway/` | Gateway HTTP (stub runner) : registre agents `.md`, runs, SSE — [features/ia-dev-service.md](./features/ia-dev-service.md) |
|
||||||
| `services/smart-ide-orchestrator/` | Routage intentions (stub forward) — [features/orchestrator-api.md](./features/orchestrator-api.md) |
|
| `services/smart-ide-orchestrator/` | Routage intentions (stub forward) — [features/orchestrator-api.md](./features/orchestrator-api.md) |
|
||||||
|
| `services/smart-ide-tools-bridge/` | API IDE : registre des services + Carbonyl / PageIndex / Chandra — [repo/service-smart-ide-tools-bridge.md](./repo/service-smart-ide-tools-bridge.md) |
|
||||||
|
| `config/` | Configuration locale agrégée (`services.local.env.example`) pour l’IDE et les services |
|
||||||
|
|
||||||
## Environnements test, pprod, prod
|
## Environnements test, pprod, prod
|
||||||
|
|
||||||
|
|||||||
4
services/agent-regex-search-api/.env.example
Normal file
4
services/agent-regex-search-api/.env.example
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
REGEX_SEARCH_HOST=127.0.0.1
|
||||||
|
REGEX_SEARCH_PORT=37143
|
||||||
|
REGEX_SEARCH_TOKEN=
|
||||||
|
REGEX_SEARCH_ROOT=
|
||||||
@ -38,6 +38,8 @@ Dans **`projects/<id>/conf.json`**, sous **`smart_ide`**, champ optionnel **`pre
|
|||||||
|
|
||||||
Le script **`scripts/open-carbonyl-preview-test.sh`** lit **`projects/active-project.json`** (ou **`SMART_IDE_PROJECT_ID`**, ou argument **`--project <id>`**) puis **`preview_urls.test`**. En secours : variable **`PREVIEW_TEST_URL`**.
|
Le script **`scripts/open-carbonyl-preview-test.sh`** lit **`projects/active-project.json`** (ou **`SMART_IDE_PROJECT_ID`**, ou argument **`--project <id>`**) puis **`preview_urls.test`**. En secours : variable **`PREVIEW_TEST_URL`**.
|
||||||
|
|
||||||
|
**API IDE** : plan d’ouverture d’URL via **`smart-ide-tools-bridge`** — `POST /v1/carbonyl/open-plan` (Bearer) — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
Documentation : [docs/repo/service-carbonyl.md](../../docs/repo/service-carbonyl.md), [docs/features/carbonyl-terminal-browser.md](../../docs/features/carbonyl-terminal-browser.md).
|
Documentation : [docs/repo/service-carbonyl.md](../../docs/repo/service-carbonyl.md), [docs/features/carbonyl-terminal-browser.md](../../docs/features/carbonyl-terminal-browser.md).
|
||||||
|
|
||||||
## Licence amont
|
## Licence amont
|
||||||
|
|||||||
@ -55,6 +55,8 @@ cd ..
|
|||||||
- **OCR / numérisation structurée** pour pipelines documentaires, en amont de **PageIndex** ([PageIndex](../pageindex/README.md)) ou d’**AnythingLLM** / **docv**.
|
- **OCR / numérisation structurée** pour pipelines documentaires, en amont de **PageIndex** ([PageIndex](../pageindex/README.md)) ou d’**AnythingLLM** / **docv**.
|
||||||
- **Pas de service HTTP** dans ce dépôt : exécution **CLI** (comme **`services/pageindex/`**).
|
- **Pas de service HTTP** dans ce dépôt : exécution **CLI** (comme **`services/pageindex/`**).
|
||||||
|
|
||||||
|
**API IDE** : OCR via **`smart-ide-tools-bridge`** — `POST /v1/chandra/ocr` — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
Documentation : [docs/repo/service-chandra.md](../../docs/repo/service-chandra.md), [docs/features/chandra-ocr-documents.md](../../docs/features/chandra-ocr-documents.md).
|
Documentation : [docs/repo/service-chandra.md](../../docs/repo/service-chandra.md), [docs/features/chandra-ocr-documents.md](../../docs/features/chandra-ocr-documents.md).
|
||||||
|
|
||||||
## Ressources amont
|
## Ressources amont
|
||||||
|
|||||||
4
services/claw-harness-api/proxy/.env.example
Normal file
4
services/claw-harness-api/proxy/.env.example
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
CLAW_PROXY_HOST=127.0.0.1
|
||||||
|
CLAW_PROXY_PORT=37142
|
||||||
|
CLAW_PROXY_TOKEN=
|
||||||
|
CLAW_UPSTREAM_URL=
|
||||||
4
services/ia-dev-gateway/.env.example
Normal file
4
services/ia-dev-gateway/.env.example
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
IA_DEV_GATEWAY_HOST=127.0.0.1
|
||||||
|
IA_DEV_GATEWAY_PORT=37144
|
||||||
|
IA_DEV_GATEWAY_TOKEN=
|
||||||
|
IA_DEV_ROOT=
|
||||||
4
services/langextract-api/.env.example
Normal file
4
services/langextract-api/.env.example
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
LANGEXTRACT_API_HOST=127.0.0.1
|
||||||
|
LANGEXTRACT_API_PORT=37141
|
||||||
|
LANGEXTRACT_SERVICE_TOKEN=
|
||||||
|
LANGEXTRACT_API_KEY=
|
||||||
@ -41,6 +41,8 @@ Les options (`--model`, `--toc-check-pages`, etc.) sont celles documentées dans
|
|||||||
- **Définition sémantique structurée** des documents (arbre de sections, résumés de nœuds) pour outillage, agents ou pipelines **hors** AnythingLLM vectoriel.
|
- **Définition sémantique structurée** des documents (arbre de sections, résumés de nœuds) pour outillage, agents ou pipelines **hors** AnythingLLM vectoriel.
|
||||||
- Complément possible à la mémoire documentaire **AnythingLLM** ([anythingllm-workspaces.md](../../docs/anythingllm-workspaces.md)) : PageIndex ne remplace pas l’ingestion RAG classique ; il fournit un **index explicable** pour navigation et raisonnement.
|
- Complément possible à la mémoire documentaire **AnythingLLM** ([anythingllm-workspaces.md](../../docs/anythingllm-workspaces.md)) : PageIndex ne remplace pas l’ingestion RAG classique ; il fournit un **index explicable** pour navigation et raisonnement.
|
||||||
|
|
||||||
|
**API IDE** : exécution index via **`smart-ide-tools-bridge`** — `POST /v1/pageindex/run` — [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
Documentation : [docs/repo/service-pageindex.md](../../docs/repo/service-pageindex.md), [docs/features/pageindex-semantic-documents.md](../../docs/features/pageindex-semantic-documents.md).
|
Documentation : [docs/repo/service-pageindex.md](../../docs/repo/service-pageindex.md), [docs/features/pageindex-semantic-documents.md](../../docs/features/pageindex-semantic-documents.md).
|
||||||
|
|
||||||
## Ressources amont
|
## Ressources amont
|
||||||
|
|||||||
5
services/repos-devtools-server/.env.example
Normal file
5
services/repos-devtools-server/.env.example
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
REPOS_DEVTOOLS_HOST=127.0.0.1
|
||||||
|
REPOS_DEVTOOLS_PORT=37140
|
||||||
|
REPOS_DEVTOOLS_TOKEN=
|
||||||
|
# Git clone root (must exist)
|
||||||
|
REPOS_DEVTOOLS_ROOT=
|
||||||
12
services/smart-ide-orchestrator/.env.example
Normal file
12
services/smart-ide-orchestrator/.env.example
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
ORCHESTRATOR_HOST=127.0.0.1
|
||||||
|
ORCHESTRATOR_PORT=37145
|
||||||
|
ORCHESTRATOR_TOKEN=
|
||||||
|
|
||||||
|
OLLAMA_URL=http://127.0.0.1:11434
|
||||||
|
ANYTHINGLLM_BASE_URL=
|
||||||
|
REPOS_DEVTOOLS_URL=http://127.0.0.1:37140
|
||||||
|
REGEX_SEARCH_URL=http://127.0.0.1:37143
|
||||||
|
LANGEXTRACT_URL=http://127.0.0.1:37141
|
||||||
|
LOCAL_OFFICE_URL=http://127.0.0.1:8000
|
||||||
|
IA_DEV_GATEWAY_URL=http://127.0.0.1:37144
|
||||||
|
TOOLS_BRIDGE_URL=http://127.0.0.1:37147
|
||||||
@ -36,6 +36,9 @@ const localOfficeUrl = (): string =>
|
|||||||
const iaDevGatewayUrl = (): string =>
|
const iaDevGatewayUrl = (): string =>
|
||||||
(process.env.IA_DEV_GATEWAY_URL ?? "http://127.0.0.1:37144").replace(/\/+$/, "");
|
(process.env.IA_DEV_GATEWAY_URL ?? "http://127.0.0.1:37144").replace(/\/+$/, "");
|
||||||
|
|
||||||
|
const toolsBridgeUrl = (): string =>
|
||||||
|
(process.env.TOOLS_BRIDGE_URL ?? "http://127.0.0.1:37147").replace(/\/+$/, "");
|
||||||
|
|
||||||
const resolveIntent = (intent: string): Resolution => {
|
const resolveIntent = (intent: string): Resolution => {
|
||||||
switch (intent) {
|
switch (intent) {
|
||||||
case "code.complete":
|
case "code.complete":
|
||||||
@ -94,6 +97,50 @@ const resolveIntent = (intent: string): Resolution => {
|
|||||||
action: "post_run",
|
action: "post_run",
|
||||||
upstream: { method: "POST", url: `${iaDevGatewayUrl()}/v1/runs`, headersHint: ["Authorization", "Content-Type"] },
|
upstream: { method: "POST", url: `${iaDevGatewayUrl()}/v1/runs`, headersHint: ["Authorization", "Content-Type"] },
|
||||||
};
|
};
|
||||||
|
case "tools.registry":
|
||||||
|
return {
|
||||||
|
resolved: true,
|
||||||
|
target: "service",
|
||||||
|
action: "tools_bridge_registry",
|
||||||
|
upstream: {
|
||||||
|
method: "GET",
|
||||||
|
url: `${toolsBridgeUrl()}/v1/registry`,
|
||||||
|
headersHint: ["Authorization"],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
case "tools.carbonyl.plan":
|
||||||
|
return {
|
||||||
|
resolved: true,
|
||||||
|
target: "service",
|
||||||
|
action: "tools_carbonyl_open_plan",
|
||||||
|
upstream: {
|
||||||
|
method: "POST",
|
||||||
|
url: `${toolsBridgeUrl()}/v1/carbonyl/open-plan`,
|
||||||
|
headersHint: ["Authorization", "Content-Type"],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
case "tools.pageindex.run":
|
||||||
|
return {
|
||||||
|
resolved: true,
|
||||||
|
target: "service",
|
||||||
|
action: "tools_pageindex_run",
|
||||||
|
upstream: {
|
||||||
|
method: "POST",
|
||||||
|
url: `${toolsBridgeUrl()}/v1/pageindex/run`,
|
||||||
|
headersHint: ["Authorization", "Content-Type"],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
case "tools.chandra.ocr":
|
||||||
|
return {
|
||||||
|
resolved: true,
|
||||||
|
target: "service",
|
||||||
|
action: "tools_chandra_ocr",
|
||||||
|
upstream: {
|
||||||
|
method: "POST",
|
||||||
|
url: `${toolsBridgeUrl()}/v1/chandra/ocr`,
|
||||||
|
headersHint: ["Authorization", "Content-Type"],
|
||||||
|
},
|
||||||
|
};
|
||||||
default:
|
default:
|
||||||
return { resolved: false, reason: `Unknown intent: ${intent}` };
|
return { resolved: false, reason: `Unknown intent: ${intent}` };
|
||||||
}
|
}
|
||||||
|
|||||||
14
services/smart-ide-tools-bridge/.env.example
Normal file
14
services/smart-ide-tools-bridge/.env.example
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
# Copy to a local env file or export before npm start (do not commit secrets).
|
||||||
|
TOOLS_BRIDGE_HOST=127.0.0.1
|
||||||
|
TOOLS_BRIDGE_PORT=37147
|
||||||
|
TOOLS_BRIDGE_TOKEN=
|
||||||
|
TOOLS_BRIDGE_JOB_TIMEOUT_MS=3600000
|
||||||
|
|
||||||
|
# Path checks for POST /v1/pageindex/run and /v1/chandra/ocr
|
||||||
|
SMART_IDE_MONOREPO_ROOT=
|
||||||
|
# Optional comma-separated extra roots
|
||||||
|
# TOOLS_ALLOWED_PATH_PREFIXES=
|
||||||
|
|
||||||
|
# Carbonyl open-plan response
|
||||||
|
CARBONYL_DOCKER_IMAGE=fathyb/carbonyl
|
||||||
|
CARBONYL_RUNNER=docker
|
||||||
2
services/smart-ide-tools-bridge/.gitignore
vendored
Normal file
2
services/smart-ide-tools-bridge/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
node_modules/
|
||||||
|
dist/
|
||||||
25
services/smart-ide-tools-bridge/README.md
Normal file
25
services/smart-ide-tools-bridge/README.md
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
# smart-ide-tools-bridge
|
||||||
|
|
||||||
|
HTTP local (**Bearer**) : **registre** des URLs des micro-services + API pour outils basés sur sous-modules (**Carbonyl**, **PageIndex**, **Chandra**).
|
||||||
|
|
||||||
|
Voir [docs/repo/service-smart-ide-tools-bridge.md](../../docs/repo/service-smart-ide-tools-bridge.md) et [docs/API/smart-ide-tools-bridge-api.md](../../docs/API/smart-ide-tools-bridge-api.md).
|
||||||
|
|
||||||
|
Configuration agrégée : [config/services.local.env.example](../../config/services.local.env.example).
|
||||||
|
|
||||||
|
## Run
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd services/smart-ide-tools-bridge
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
export TOOLS_BRIDGE_TOKEN='…'
|
||||||
|
export SMART_IDE_MONOREPO_ROOT="$(cd ../.. && pwd)"
|
||||||
|
npm start
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prérequis jobs
|
||||||
|
|
||||||
|
- **PageIndex** : Python / venv dans `services/pageindex/upstream` (voir `services/pageindex/README.md`).
|
||||||
|
- **Chandra** : install HF ou vLLM dans `services/chandra` (voir `services/chandra/README.md`).
|
||||||
|
|
||||||
|
Les chemins passés aux POST doivent résoudre sous **`SMART_IDE_MONOREPO_ROOT`** (ou préfixes **`TOOLS_ALLOWED_PATH_PREFIXES`**).
|
||||||
51
services/smart-ide-tools-bridge/package-lock.json
generated
Normal file
51
services/smart-ide-tools-bridge/package-lock.json
generated
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
{
|
||||||
|
"name": "@4nk/smart-ide-tools-bridge",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {
|
||||||
|
"": {
|
||||||
|
"name": "@4nk/smart-ide-tools-bridge",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"license": "MIT",
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.0",
|
||||||
|
"typescript": "^5.3.3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@types/node": {
|
||||||
|
"version": "20.19.39",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.39.tgz",
|
||||||
|
"integrity": "sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"undici-types": "~6.21.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/typescript": {
|
||||||
|
"version": "5.9.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||||
|
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"bin": {
|
||||||
|
"tsc": "bin/tsc",
|
||||||
|
"tsserver": "bin/tsserver"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=14.17"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/undici-types": {
|
||||||
|
"version": "6.21.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||||
|
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
20
services/smart-ide-tools-bridge/package.json
Normal file
20
services/smart-ide-tools-bridge/package.json
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"name": "@4nk/smart-ide-tools-bridge",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"description": "HTTP bridge: IDE-facing API for Carbonyl, PageIndex, Chandra + service registry.",
|
||||||
|
"license": "MIT",
|
||||||
|
"type": "module",
|
||||||
|
"main": "dist/server.js",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc -p ./",
|
||||||
|
"start": "node dist/server.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.0",
|
||||||
|
"typescript": "^5.3.3"
|
||||||
|
}
|
||||||
|
}
|
||||||
21
services/smart-ide-tools-bridge/src/auth.ts
Normal file
21
services/smart-ide-tools-bridge/src/auth.ts
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||||
|
|
||||||
|
export const readExpectedToken = (): string => {
|
||||||
|
return process.env.TOOLS_BRIDGE_TOKEN?.trim() ?? "";
|
||||||
|
};
|
||||||
|
|
||||||
|
export const requireBearer = (
|
||||||
|
req: IncomingMessage,
|
||||||
|
res: ServerResponse,
|
||||||
|
expected: string,
|
||||||
|
): boolean => {
|
||||||
|
const h = req.headers.authorization ?? "";
|
||||||
|
const match = /^Bearer\s+(.+)$/i.exec(h);
|
||||||
|
const got = match?.[1]?.trim() ?? "";
|
||||||
|
if (got !== expected) {
|
||||||
|
res.writeHead(401, { "Content-Type": "application/json; charset=utf-8" });
|
||||||
|
res.end(JSON.stringify({ error: "Unauthorized" }));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
};
|
||||||
13
services/smart-ide-tools-bridge/src/httpUtil.ts
Normal file
13
services/smart-ide-tools-bridge/src/httpUtil.ts
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
import type { IncomingMessage } from "node:http";
|
||||||
|
|
||||||
|
export const readJsonBody = async (req: IncomingMessage): Promise<unknown> => {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
for await (const chunk of req) {
|
||||||
|
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
||||||
|
}
|
||||||
|
const raw = Buffer.concat(chunks).toString("utf-8").trim();
|
||||||
|
if (raw.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
return JSON.parse(raw) as unknown;
|
||||||
|
};
|
||||||
49
services/smart-ide-tools-bridge/src/paths.ts
Normal file
49
services/smart-ide-tools-bridge/src/paths.ts
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
import path from "node:path";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
export const defaultMonorepoRoot = (): string => {
|
||||||
|
const fromEnv = process.env.SMART_IDE_MONOREPO_ROOT?.trim();
|
||||||
|
if (fromEnv) {
|
||||||
|
return path.resolve(fromEnv);
|
||||||
|
}
|
||||||
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
return path.resolve(here, "..", "..", "..");
|
||||||
|
};
|
||||||
|
|
||||||
|
const extraPrefixes = (): string[] => {
|
||||||
|
const raw = process.env.TOOLS_ALLOWED_PATH_PREFIXES?.trim();
|
||||||
|
if (!raw) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
.split(",")
|
||||||
|
.map((s) => path.resolve(s.trim()))
|
||||||
|
.filter((s) => s.length > 0);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isPathAllowed = (absPath: string, monorepoRoot: string): boolean => {
|
||||||
|
const norm = path.resolve(absPath);
|
||||||
|
const roots = [path.resolve(monorepoRoot), ...extraPrefixes()];
|
||||||
|
return roots.some((r) => norm === r || norm.startsWith(r + path.sep));
|
||||||
|
};
|
||||||
|
|
||||||
|
export const assertAllowedFile = (p: string, monorepoRoot: string): string => {
|
||||||
|
const abs = path.resolve(p);
|
||||||
|
if (!isPathAllowed(abs, monorepoRoot)) {
|
||||||
|
throw new Error(`Path not allowed under SMART_IDE_MONOREPO_ROOT: ${abs}`);
|
||||||
|
}
|
||||||
|
if (!fs.existsSync(abs)) {
|
||||||
|
throw new Error(`Path does not exist: ${abs}`);
|
||||||
|
}
|
||||||
|
return abs;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Output directory (may not exist yet); must resolve under an allowed root. */
|
||||||
|
export const assertAllowedDirPath = (p: string, monorepoRoot: string): string => {
|
||||||
|
const abs = path.resolve(p);
|
||||||
|
if (!isPathAllowed(abs, monorepoRoot)) {
|
||||||
|
throw new Error(`Path not allowed under SMART_IDE_MONOREPO_ROOT: ${abs}`);
|
||||||
|
}
|
||||||
|
return abs;
|
||||||
|
};
|
||||||
261
services/smart-ide-tools-bridge/src/server.ts
Normal file
261
services/smart-ide-tools-bridge/src/server.ts
Normal file
@ -0,0 +1,261 @@
|
|||||||
|
import * as http from "node:http";
|
||||||
|
import path from "node:path";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import { readExpectedToken, requireBearer } from "./auth.js";
|
||||||
|
import { readJsonBody } from "./httpUtil.js";
|
||||||
|
import {
|
||||||
|
assertAllowedDirPath,
|
||||||
|
assertAllowedFile,
|
||||||
|
defaultMonorepoRoot,
|
||||||
|
} from "./paths.js";
|
||||||
|
import { runProcess } from "./spawnJob.js";
|
||||||
|
|
||||||
|
const HOST = process.env.TOOLS_BRIDGE_HOST ?? "127.0.0.1";
|
||||||
|
const PORT = Number(process.env.TOOLS_BRIDGE_PORT ?? "37147");
|
||||||
|
const JOB_TIMEOUT_MS = Number(
|
||||||
|
process.env.TOOLS_BRIDGE_JOB_TIMEOUT_MS ?? "3600000",
|
||||||
|
);
|
||||||
|
|
||||||
|
const json = (res: http.ServerResponse, status: number, body: unknown): void => {
|
||||||
|
res.writeHead(status, { "Content-Type": "application/json; charset=utf-8" });
|
||||||
|
res.end(JSON.stringify(body));
|
||||||
|
};
|
||||||
|
|
||||||
|
const isRecord = (v: unknown): v is Record<string, unknown> =>
|
||||||
|
typeof v === "object" && v !== null && !Array.isArray(v);
|
||||||
|
|
||||||
|
const safeHttpUrl = (raw: string): string => {
|
||||||
|
const u = new URL(raw);
|
||||||
|
if (u.protocol !== "http:" && u.protocol !== "https:") {
|
||||||
|
throw new Error("Only http and https URLs are allowed");
|
||||||
|
}
|
||||||
|
return u.toString();
|
||||||
|
};
|
||||||
|
|
||||||
|
const serviceRegistry = (): Record<string, unknown> => ({
|
||||||
|
monorepoRoot: defaultMonorepoRoot(),
|
||||||
|
services: {
|
||||||
|
tools_bridge: {
|
||||||
|
url: `http://${HOST}:${PORT}`,
|
||||||
|
tokenEnv: "TOOLS_BRIDGE_TOKEN",
|
||||||
|
},
|
||||||
|
repos_devtools: {
|
||||||
|
url: (process.env.REPOS_DEVTOOLS_URL ?? `http://127.0.0.1:${process.env.REPOS_DEVTOOLS_PORT ?? "37140"}`).replace(
|
||||||
|
/\/+$/,
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
tokenEnv: "REPOS_DEVTOOLS_TOKEN",
|
||||||
|
},
|
||||||
|
langextract: {
|
||||||
|
url: `http://${process.env.LANGEXTRACT_API_HOST ?? "127.0.0.1"}:${process.env.LANGEXTRACT_API_PORT ?? "37141"}`,
|
||||||
|
tokenEnv: "LANGEXTRACT_SERVICE_TOKEN",
|
||||||
|
},
|
||||||
|
claw_proxy: {
|
||||||
|
url: `http://${process.env.CLAW_PROXY_HOST ?? "127.0.0.1"}:${process.env.CLAW_PROXY_PORT ?? "37142"}`,
|
||||||
|
tokenEnv: "CLAW_PROXY_TOKEN",
|
||||||
|
},
|
||||||
|
regex_search: {
|
||||||
|
url: `http://${process.env.REGEX_SEARCH_HOST ?? "127.0.0.1"}:${process.env.REGEX_SEARCH_PORT ?? "37143"}`,
|
||||||
|
tokenEnv: "REGEX_SEARCH_TOKEN",
|
||||||
|
},
|
||||||
|
local_office: {
|
||||||
|
url: (process.env.LOCAL_OFFICE_URL ?? "http://127.0.0.1:8000").replace(/\/+$/, ""),
|
||||||
|
tokenEnv: "LOCAL_OFFICE_API_KEY",
|
||||||
|
header: "X-API-Key",
|
||||||
|
},
|
||||||
|
ia_dev_gateway: {
|
||||||
|
url: `http://${process.env.IA_DEV_GATEWAY_HOST ?? "127.0.0.1"}:${process.env.IA_DEV_GATEWAY_PORT ?? "37144"}`,
|
||||||
|
tokenEnv: "IA_DEV_GATEWAY_TOKEN",
|
||||||
|
},
|
||||||
|
orchestrator: {
|
||||||
|
url: `http://${process.env.ORCHESTRATOR_HOST ?? "127.0.0.1"}:${process.env.ORCHESTRATOR_PORT ?? "37145"}`,
|
||||||
|
tokenEnv: "ORCHESTRATOR_TOKEN",
|
||||||
|
},
|
||||||
|
anythingllm_devtools: {
|
||||||
|
url: `http://${process.env.ANYTHINGLLM_DEVTOOLS_HOST ?? "127.0.0.1"}:${process.env.ANYTHINGLLM_DEVTOOLS_PORT ?? "37146"}`,
|
||||||
|
tokenEnv: "ANYTHINGLLM_DEVTOOLS_TOKEN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
submodule_tools: {
|
||||||
|
carbonyl: {
|
||||||
|
openPlanPath: "/v1/carbonyl/open-plan",
|
||||||
|
upstreamSubmodule: "services/carbonyl/upstream",
|
||||||
|
},
|
||||||
|
pageindex: {
|
||||||
|
runPath: "/v1/pageindex/run",
|
||||||
|
upstreamSubmodule: "services/pageindex/upstream",
|
||||||
|
},
|
||||||
|
chandra: {
|
||||||
|
ocrPath: "/v1/chandra/ocr",
|
||||||
|
upstreamSubmodule: "services/chandra/upstream",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const main = (): void => {
|
||||||
|
const token = readExpectedToken();
|
||||||
|
if (token.length === 0) {
|
||||||
|
console.error("smart-ide-tools-bridge: set TOOLS_BRIDGE_TOKEN (non-empty secret).");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const monorepoRoot = defaultMonorepoRoot();
|
||||||
|
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const url = new URL(req.url ?? "/", `http://${HOST}`);
|
||||||
|
const p = url.pathname;
|
||||||
|
|
||||||
|
if (req.method === "GET" && (p === "/health" || p === "/health/")) {
|
||||||
|
json(res, 200, { status: "ok", service: "smart-ide-tools-bridge" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "GET" && p === "/v1/registry") {
|
||||||
|
if (!requireBearer(req, res, token)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
json(res, 200, serviceRegistry());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "POST" && p === "/v1/carbonyl/open-plan") {
|
||||||
|
if (!requireBearer(req, res, token)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const body = await readJsonBody(req);
|
||||||
|
if (!isRecord(body) || typeof body.url !== "string") {
|
||||||
|
json(res, 422, { error: "Missing url (string)" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const targetUrl = safeHttpUrl(body.url);
|
||||||
|
const image = process.env.CARBONYL_DOCKER_IMAGE ?? "fathyb/carbonyl";
|
||||||
|
const runner = (process.env.CARBONYL_RUNNER ?? "docker").toLowerCase();
|
||||||
|
if (runner === "native") {
|
||||||
|
json(res, 200, {
|
||||||
|
runner: "native",
|
||||||
|
argv: ["carbonyl", targetUrl],
|
||||||
|
hint: "Run in a terminal; requires carbonyl in PATH.",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
json(res, 200, {
|
||||||
|
runner: "docker",
|
||||||
|
argv: ["docker", "run", "--rm", "-ti", image, targetUrl],
|
||||||
|
hint: "Interactive TTY (-ti) required; run from a terminal session.",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "POST" && p === "/v1/pageindex/run") {
|
||||||
|
if (!requireBearer(req, res, token)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const body = await readJsonBody(req);
|
||||||
|
if (!isRecord(body) || typeof body.inputPath !== "string") {
|
||||||
|
json(res, 422, { error: "Missing inputPath (string)" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const mode =
|
||||||
|
body.mode === "md" ? "md" : body.mode === "pdf" ? "pdf" : null;
|
||||||
|
if (!mode) {
|
||||||
|
json(res, 422, { error: "mode must be pdf or md" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const inputAbs = assertAllowedFile(body.inputPath, monorepoRoot);
|
||||||
|
const pageindexDir = path.join(monorepoRoot, "services", "pageindex");
|
||||||
|
const script = path.join(pageindexDir, "run-pageindex.sh");
|
||||||
|
if (!fs.existsSync(script)) {
|
||||||
|
json(res, 500, { error: "run-pageindex.sh missing" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const flag = mode === "pdf" ? "--pdf_path" : "--md_path";
|
||||||
|
const { code, stdout, stderr } = await runProcess(script, [flag, inputAbs], {
|
||||||
|
cwd: pageindexDir,
|
||||||
|
timeoutMs: JOB_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
const base = path.basename(inputAbs, path.extname(inputAbs));
|
||||||
|
const resultPath = path.join(
|
||||||
|
monorepoRoot,
|
||||||
|
"services",
|
||||||
|
"pageindex",
|
||||||
|
"upstream",
|
||||||
|
"results",
|
||||||
|
`${base}_structure.json`,
|
||||||
|
);
|
||||||
|
json(res, code === 0 ? 200 : 500, {
|
||||||
|
ok: code === 0,
|
||||||
|
exitCode: code,
|
||||||
|
stdoutTail: stdout.slice(-8000),
|
||||||
|
stderrTail: stderr.slice(-8000),
|
||||||
|
resultPath: fs.existsSync(resultPath) ? resultPath : null,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === "POST" && p === "/v1/chandra/ocr") {
|
||||||
|
if (!requireBearer(req, res, token)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const body = await readJsonBody(req);
|
||||||
|
if (
|
||||||
|
!isRecord(body) ||
|
||||||
|
typeof body.inputPath !== "string" ||
|
||||||
|
typeof body.outputPath !== "string"
|
||||||
|
) {
|
||||||
|
json(res, 422, {
|
||||||
|
error: "Missing inputPath (string) or outputPath (string)",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const inputAbs = assertAllowedFile(body.inputPath, monorepoRoot);
|
||||||
|
const outputAbs = assertAllowedDirPath(body.outputPath, monorepoRoot);
|
||||||
|
fs.mkdirSync(outputAbs, { recursive: true });
|
||||||
|
const method =
|
||||||
|
body.method === "vllm" ? "vllm" : body.method === "hf" ? "hf" : "hf";
|
||||||
|
const chandraDir = path.join(monorepoRoot, "services", "chandra");
|
||||||
|
const runHf = path.join(chandraDir, "run-chandra-hf.sh");
|
||||||
|
const runAny = path.join(chandraDir, "run-chandra.sh");
|
||||||
|
if (!fs.existsSync(runAny)) {
|
||||||
|
json(res, 500, { error: "Chandra run script missing" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const script =
|
||||||
|
method === "hf" && fs.existsSync(runHf)
|
||||||
|
? runHf
|
||||||
|
: runAny;
|
||||||
|
const args =
|
||||||
|
method === "hf" && script === runHf
|
||||||
|
? [inputAbs, outputAbs]
|
||||||
|
: method === "hf"
|
||||||
|
? [inputAbs, outputAbs, "--method", "hf"]
|
||||||
|
: [inputAbs, outputAbs, "--method", "vllm"];
|
||||||
|
const { code, stdout, stderr } = await runProcess(script, args, {
|
||||||
|
cwd: chandraDir,
|
||||||
|
timeoutMs: JOB_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
json(res, code === 0 ? 200 : 500, {
|
||||||
|
ok: code === 0,
|
||||||
|
exitCode: code,
|
||||||
|
outputDir: outputAbs,
|
||||||
|
stdoutTail: stdout.slice(-8000),
|
||||||
|
stderrTail: stderr.slice(-8000),
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
json(res, 404, { error: "Not found" });
|
||||||
|
} catch (e) {
|
||||||
|
const msg = e instanceof Error ? e.message : String(e);
|
||||||
|
json(res, 400, { error: msg });
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(PORT, HOST, () => {
|
||||||
|
console.error(`smart-ide-tools-bridge listening on http://${HOST}:${PORT}`);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
main();
|
||||||
35
services/smart-ide-tools-bridge/src/spawnJob.ts
Normal file
35
services/smart-ide-tools-bridge/src/spawnJob.ts
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
import { spawn } from "node:child_process";
|
||||||
|
|
||||||
|
export const runProcess = (
|
||||||
|
command: string,
|
||||||
|
args: string[],
|
||||||
|
options: { cwd: string; timeoutMs: number },
|
||||||
|
): Promise<{ code: number | null; stdout: string; stderr: string }> => {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(command, args, {
|
||||||
|
cwd: options.cwd,
|
||||||
|
env: process.env,
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
const t = setTimeout(() => {
|
||||||
|
child.kill("SIGTERM");
|
||||||
|
reject(new Error(`Process timeout after ${options.timeoutMs}ms`));
|
||||||
|
}, options.timeoutMs);
|
||||||
|
child.stdout?.on("data", (d: Buffer) => {
|
||||||
|
stdout += d.toString("utf-8");
|
||||||
|
});
|
||||||
|
child.stderr?.on("data", (d: Buffer) => {
|
||||||
|
stderr += d.toString("utf-8");
|
||||||
|
});
|
||||||
|
child.on("close", (code) => {
|
||||||
|
clearTimeout(t);
|
||||||
|
resolve({ code, stdout, stderr });
|
||||||
|
});
|
||||||
|
child.on("error", (e) => {
|
||||||
|
clearTimeout(t);
|
||||||
|
reject(e);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
16
services/smart-ide-tools-bridge/tsconfig.json
Normal file
16
services/smart-ide-tools-bridge/tsconfig.json
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "NodeNext",
|
||||||
|
"moduleResolution": "NodeNext",
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src",
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"noImplicitReturns": true,
|
||||||
|
"noUnusedLocals": true,
|
||||||
|
"noUnusedParameters": true,
|
||||||
|
"declaration": false
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"]
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user