#!/usr/bin/env node /** * Upload files changed between ORIG_HEAD and HEAD to AnythingLLM (post-merge / after pull). * Requires: ANYTHINGLLM_BASE_URL, ANYTHINGLLM_API_KEY, workspace slug via ANYTHINGLLM_WORKSPACE_SLUG or .anythingllm.json */ import { execFileSync } from "node:child_process"; import * as fs from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import ignore from "ignore"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ALWAYS_IGNORE = [".git/", "node_modules/", "**/node_modules/"].join("\n"); const readJson = (p) => { const raw = fs.readFileSync(p, "utf8"); return JSON.parse(raw); }; const git = (repoRoot, args) => { return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], }).trim(); }; const parseArgs = () => { const out = { repoRoot: process.cwd() }; const argv = process.argv.slice(2); for (let i = 0; i < argv.length; i += 1) { if (argv[i] === "--repo-root" && argv[i + 1]) { out.repoRoot = path.resolve(argv[i + 1]); i += 1; } } return out; }; const loadWorkspaceSlug = (repoRoot) => { const env = process.env.ANYTHINGLLM_WORKSPACE_SLUG?.trim(); if (env) { return env; } const cfgPath = path.join(repoRoot, ".anythingllm.json"); try { const j = readJson(cfgPath); if (typeof j.workspaceSlug === "string" && j.workspaceSlug.trim().length > 0) { return j.workspaceSlug.trim(); } } catch { /* missing */ } return ""; }; const normalizeApiKey = (raw) => { const t = raw.trim(); const m = /^Bearer\s+/i.exec(t); return m ? t.slice(m[0].length).trim() : t; }; const uploadOne = async (baseUrl, apiKey, slug, absPath, uploadName) => { const root = baseUrl.replace(/\/+$/, ""); const buf = await fsPromises.readFile(absPath); const body = new FormData(); body.append("file", new Blob([buf]), uploadName); body.append("addToWorkspaces", slug); const res = await fetch(`${root}/api/v1/document/upload`, { method: "POST", headers: { Authorization: `Bearer ${apiKey}` }, body, }); const text = await res.text(); let parsed; try { parsed = JSON.parse(text); } catch { throw new Error(`non-JSON ${res.status}: ${text.slice(0, 200)}`); } if (!res.ok || parsed.success !== true) { throw new Error(`${res.status}: ${text.slice(0, 400)}`); } }; const main = async () => { const { repoRoot } = parseArgs(); const baseUrl = process.env.ANYTHINGLLM_BASE_URL?.trim() ?? ""; const apiKeyRaw = process.env.ANYTHINGLLM_API_KEY?.trim() ?? ""; const maxBytes = Number(process.env.ANYTHINGLLM_SYNC_MAX_FILE_BYTES ?? 5242880); const maxFiles = Number(process.env.ANYTHINGLLM_SYNC_MAX_FILES ?? 200); if (!baseUrl || !apiKeyRaw) { console.error( "anythingllm-pull-sync: missing ANYTHINGLLM_BASE_URL or ANYTHINGLLM_API_KEY — skip.", ); process.exit(0); } const apiKey = normalizeApiKey(apiKeyRaw); const slug = loadWorkspaceSlug(repoRoot); if (!slug) { console.error( "anythingllm-pull-sync: set ANYTHINGLLM_WORKSPACE_SLUG or .anythingllm.json { \"workspaceSlug\": \"…\" } — skip.", ); process.exit(0); } try { git(repoRoot, ["rev-parse", "-q", "--verify", "ORIG_HEAD"]); } catch { console.error("anythingllm-pull-sync: no ORIG_HEAD (not a merge/pull) — skip."); process.exit(0); } let names; try { const out = git(repoRoot, [ "diff", "--name-only", "--diff-filter=ACMRT", "ORIG_HEAD", "HEAD", ]); names = out.length > 0 ? out.split("\n").filter(Boolean) : []; } catch (e) { console.error("anythingllm-pull-sync: git diff failed — skip.", e.message); process.exit(0); } if (names.length === 0) { console.error("anythingllm-pull-sync: no file changes between ORIG_HEAD and HEAD."); process.exit(0); } const ignorePath = path.join(repoRoot, ".4nkaiignore"); let userRules = ""; try { userRules = await fsPromises.readFile(ignorePath, "utf8"); } catch { userRules = ""; } const ig = ignore(); ig.add(ALWAYS_IGNORE); ig.add(userRules); let uploaded = 0; let skipped = 0; const errors = []; for (const rel of names) { if (rel.includes("..") || path.isAbsolute(rel)) { skipped += 1; continue; } const posix = rel.split(path.sep).join("/"); if (ig.ignores(posix)) { skipped += 1; continue; } const abs = path.join(repoRoot, rel); let st; try { st = await fsPromises.stat(abs); } catch { skipped += 1; continue; } if (!st.isFile()) { skipped += 1; continue; } if (st.size > maxBytes) { skipped += 1; continue; } if (uploaded >= maxFiles) { console.error("anythingllm-pull-sync: cap reached (ANYTHINGLLM_SYNC_MAX_FILES)."); break; } const uploadName = posix.split("/").join("__"); try { await uploadOne(baseUrl, apiKey, slug, abs, uploadName); uploaded += 1; } catch (e) { errors.push(`${posix}: ${e instanceof Error ? e.message : String(e)}`); } } console.error( `anythingllm-pull-sync: uploaded=${uploaded} skipped=${skipped} errors=${errors.length}`, ); for (const line of errors.slice(0, 20)) { console.error(line); } if (errors.length > 20) { console.error(`… ${errors.length - 20} more`); } process.exit(0); }; main().catch((e) => { console.error("anythingllm-pull-sync:", e); process.exit(1); });