Initial state: - ia_dev was historically referenced as ./ia_dev in docs and integrations, while the vendored module lives under services/ia_dev. - AnythingLLM sync and hook installation had error masking / weak exit signaling. - Proxy layers did not validate proxy path segments, allowing path normalization tricks. Motivation: - Make the IDE-oriented workflow usable (sync -> act -> deploy/preview) with explicit errors. - Reduce security footguns in proxying and script automation. Resolution: - Standardize IA_DEV_ROOT usage and documentation to services/ia_dev. - Add SSH remote data mirroring + optional AnythingLLM ingestion. - Extend AnythingLLM pull sync to support upload-all/prefix and fail on upload errors. - Harden smart-ide-sso-gateway and smart-ide-global-api proxying with safe-path checks and non-leaking error responses. - Improve ia-dev-gateway runner validation and reduce sensitive path leakage. - Add site scaffold tool (Vite/React) with OIDC + chat via sso-gateway -> orchestrator. Root cause: - Historical layout changes (submodule -> vendored tree) and missing central contracts for path resolution. - Missing validation for proxy path traversal patterns. - Overuse of silent fallbacks (|| true, exit 0 on partial failures) in automation scripts. Impacted features: - Project sync: git pull + AnythingLLM sync + remote data mirror ingestion. - Site frontends: SSO gateway proxy and orchestrator intents (rag.query, chat.local). - Agent execution: ia-dev-gateway script runner and SSE output. Code modified: - scripts/remote-data-ssh-sync.sh - scripts/anythingllm-pull-sync/sync.mjs - scripts/install-anythingllm-post-merge-hook.sh - cron/git-pull-project-clones.sh - services/smart-ide-sso-gateway/src/server.ts - services/smart-ide-global-api/src/server.ts - services/smart-ide-orchestrator/src/server.ts - services/ia-dev-gateway/src/server.ts - services/ia_dev/tools/site-generate.sh Documentation modified: - docs/** (architecture, API docs, ia_dev module + integration, scripts) Configurations modified: - config/services.local.env.example - services/*/.env.example Files in deploy modified: - services/ia_dev/deploy/* Files in logs impacted: - logs/ia_dev.log (runtime only) - .logs/* (runtime only) Databases and other sources modified: - None Off-project modifications: - None Files in .smartIde modified: - .smartIde/agents/*.md - services/ia_dev/.smartIde/** Files in .secrets modified: - None New patch version in VERSION: - 0.0.5 CHANGELOG.md updated: - yes
107 lines
3.0 KiB
Bash
Executable File
107 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# For each projects/<id>/conf.json with project_path pointing to a Git clone under ../
|
|
# (or anywhere), fetch origin and fast-forward the current branch if safe.
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
PROJECTS_CONF="$ROOT/projects"
|
|
LOG_PREFIX="[git-pull-project-clones]"
|
|
|
|
usage() {
|
|
echo "Usage: $0 [--all | --project <id>]" >&2
|
|
echo " Reads projects/<id>/conf.json (jq required). Skips if cron.git_pull is false." >&2
|
|
exit 1
|
|
}
|
|
|
|
filter_id=""
|
|
if [[ "${1:-}" == "--project" && -n "${2:-}" ]]; then
|
|
filter_id="$2"
|
|
elif [[ -n "${1:-}" && "$1" != "--all" ]]; then
|
|
usage
|
|
fi
|
|
|
|
err_count=0
|
|
|
|
command -v jq >/dev/null 2>&1 || {
|
|
echo "$LOG_PREFIX jq not found; install jq." >&2
|
|
exit 1
|
|
}
|
|
|
|
pull_clone() {
|
|
local id="$1" path="$2"
|
|
if [[ ! -d "$path" ]]; then
|
|
echo "$LOG_PREFIX skip $id: project_path not a directory: $path"
|
|
return 0
|
|
fi
|
|
if [[ ! -d "$path/.git" ]]; then
|
|
echo "$LOG_PREFIX skip $id: not a git repository: $path"
|
|
return 0
|
|
fi
|
|
local branch
|
|
if ! branch="$(git -C "$path" symbolic-ref -q --short HEAD 2>/dev/null)"; then
|
|
echo "$LOG_PREFIX skip $id: detached HEAD in $path"
|
|
return 0
|
|
fi
|
|
if ! git -C "$path" remote get-url origin >/dev/null 2>&1; then
|
|
echo "$LOG_PREFIX skip $id: no origin remote in $path"
|
|
return 0
|
|
fi
|
|
echo "$LOG_PREFIX $id: fetch origin ($path, branch $branch)"
|
|
if ! git -C "$path" fetch origin "$branch" 2>&1; then
|
|
echo "$LOG_PREFIX $id: fetch failed" >&2
|
|
return 1
|
|
fi
|
|
local head oref
|
|
head="$(git -C "$path" rev-parse HEAD)"
|
|
oref="origin/$branch"
|
|
if ! git -C "$path" rev-parse --verify -q "$oref" >/dev/null 2>&1; then
|
|
echo "$LOG_PREFIX skip $id: $oref missing after fetch"
|
|
return 0
|
|
fi
|
|
local remote_sha
|
|
remote_sha="$(git -C "$path" rev-parse "$oref")"
|
|
if [[ "$head" == "$remote_sha" ]]; then
|
|
echo "$LOG_PREFIX $id: up to date"
|
|
return 0
|
|
fi
|
|
if ! git -C "$path" merge-base --is-ancestor "$head" "$remote_sha" 2>/dev/null; then
|
|
echo "$LOG_PREFIX skip $id: not fast-forward (diverged or unrelated); manual merge required"
|
|
return 0
|
|
fi
|
|
echo "$LOG_PREFIX $id: fast-forward to $oref"
|
|
git -C "$path" merge --ff-only "$oref"
|
|
echo "$LOG_PREFIX $id: done"
|
|
}
|
|
|
|
for conf in "$PROJECTS_CONF"/*/conf.json; do
|
|
[[ -f "$conf" ]] || continue
|
|
id="$(basename "$(dirname "$conf")")"
|
|
if [[ -n "$filter_id" && "$id" != "$filter_id" ]]; then
|
|
continue
|
|
fi
|
|
if [[ "$id" == "example" ]]; then
|
|
echo "$LOG_PREFIX skip $id: template project"
|
|
continue
|
|
fi
|
|
if [[ "$(jq -r '.cron.git_pull // true' "$conf")" == "false" ]]; then
|
|
echo "$LOG_PREFIX skip $id: cron.git_pull is false"
|
|
continue
|
|
fi
|
|
path="$(jq -r '.project_path // empty' "$conf")"
|
|
if [[ -z "$path" || "$path" == "null" ]]; then
|
|
echo "$LOG_PREFIX skip $id: empty project_path"
|
|
continue
|
|
fi
|
|
if [[ "$path" != /* ]]; then
|
|
path="$(cd "$ROOT" && realpath -m "$path" 2>/dev/null || echo "$ROOT/$path")"
|
|
fi
|
|
if ! pull_clone "$id" "$path"; then
|
|
err_count=$((err_count + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ "$err_count" -gt 0 ]]; then
|
|
echo "$LOG_PREFIX errors: $err_count (see output above)" >&2
|
|
exit 1
|
|
fi
|