[bug] Fix checkConnections logic

package-lock.json

@@ -1,15 +1,16 @@
 {
   "name": "sdk_signer",
-  "version": "1.0.0",
+  "version": "0.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "sdk_signer",
-      "version": "1.0.0",
+      "version": "0.1.1",
-      "license": "ISC",
+      "license": "MIT",
       "dependencies": {
         "@types/ws": "^8.5.10",
+        "axios": "^1.7.8",
         "dotenv": "^16.3.1",
         "level": "^10.0.0",
         "ws": "^8.14.2"
@@ -941,6 +942,21 @@
         "node": "*"
       }
     },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz",
+      "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==",
+      "dependencies": {
+        "follow-redirects": "^1.15.6",
+        "form-data": "^4.0.4",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -1001,6 +1017,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/chai": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz",
@@ -1048,6 +1076,17 @@
         "node": ">=18"
       }
     },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/confbox": {
       "version": "0.1.8",
       "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz",
@@ -1107,6 +1146,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/diff": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -1137,6 +1184,60 @@
         "url": "https://dotenvx.com"
       }
     },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-object-atoms": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
+      "dependencies": {
+        "es-errors": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/esbuild": {
       "version": "0.21.5",
       "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
@@ -1210,6 +1311,40 @@
         "url": "https://github.com/sindresorhus/execa?sponsor=1"
       }
     },
+    "node_modules/follow-redirects": {
+      "version": "1.15.11",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
+      "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/fsevents": {
       "version": "2.3.3",
       "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@@ -1225,6 +1360,14 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
+    "node_modules/function-bind": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/get-func-name": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@@ -1235,6 +1378,41 @@
         "node": "*"
       }
     },
+    "node_modules/get-intrinsic": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
+        "function-bind": "^1.1.2",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/get-stream": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz",
@@ -1248,6 +1426,53 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/hasown": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "dependencies": {
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/human-signals": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz",
@@ -1406,6 +1631,14 @@
       "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
       "dev": true
     },
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/maybe-combine-errors": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz",
@@ -1421,6 +1654,25 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/mimic-fn": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz",
@@ -1661,6 +1913,11 @@
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "node_modules/react-is": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",

package.json

@@ -20,6 +20,7 @@
     "@types/node": "^22.5.0"
   },
   "dependencies": {
+    "axios": "^1.7.8",
     "ws": "^8.14.2",
     "@types/ws": "^8.5.10",
     "dotenv": "^16.3.1",

src/database.service.ts

@@ -60,19 +60,27 @@ export default class Database {
   }
 
   private parseKey(fullKey: string): { storeName: string; key: string } | null {
-    const parts = fullKey.split(':', 2);
+    const colonIndex = fullKey.indexOf(':');
-    if (parts.length !== 2) return null;
+    if (colonIndex === -1) return null;
-    return { storeName: parts[0], key: parts[1] };
+    
+    const storeName = fullKey.substring(0, colonIndex);
+    const key = fullKey.substring(colonIndex + 1);
+    
+    return { storeName, key };
   }
 
   /**
    * Get a single object from a store
    * O(log n) operation - only reads specific key
    */
-  public async getObject(storeName: string, key: string): Promise<any | null> {
+  public async getObject(storeName: string, key: string, isBuffer: boolean = false): Promise<any | null> {
     try {
       const fullKey = this.getKey(storeName, key);
-      return await this.db.get(fullKey);
+      if (isBuffer) {
+        return await this.db.get(fullKey, { valueEncoding: 'buffer' });
+      } else {
+        return await this.db.get(fullKey);
+      }
     } catch (error) {
       if ((error as any).code === 'LEVEL_NOT_FOUND') {
         return null;
@@ -85,12 +93,16 @@ export default class Database {
    * Add or update an object in a store
    * O(log n) operation - only writes specific key-value pair
    */
-  public async addObject(operation: DatabaseObject): Promise<void> {
+  public async addObject(operation: DatabaseObject, isBuffer: boolean = false): Promise<void> {
     const { storeName, object, key } = operation;
     
     if (key) {
       const fullKey = this.getKey(storeName, key);
-      await this.db.put(fullKey, object);
+      if (isBuffer) {
+        await this.db.put(fullKey, object, { valueEncoding: 'buffer' });
+      } else {
+        await this.db.put(fullKey, object);
+      }
     } else {
       // Auto-generate key if none provided
       const autoKey = Date.now().toString() + Math.random().toString(36).substr(2, 9);

src/relay-manager.ts

@@ -310,7 +310,13 @@ export class RelayManager {
 
   // Relay Message Handling
   private handleRelayMessage(relayId: string, message: any): void {
-    console.log(`📨 Received message from relay ${relayId}:`, message);
+    console.log(`📨 Received message from relay ${relayId}:`);
+
+    if (message.flag === 'Handshake') {
+      console.log('🔑 Handshake message');
+    } else {
+      console.log(`🔑 ${message.flag} message: ${message.content}`);
+    }
     
     // Handle different types of relay responses
     if (message.flag) {

src/service.ts

@@ -5,6 +5,7 @@ import { ApiReturn, Device, HandshakeMessage, Member, MerkleProofResult, OutPoin
 import { RelayManager } from './relay-manager';
 import { config } from './config';
 import { EMPTY32BYTES } from './utils';
+import { storeData } from './storage.service';
 
 const DEFAULTAMOUNT = 1000n;
 const DEVICE_KEY = 'main_device';
@@ -80,56 +81,49 @@ export class Service {
             const existing = await this.getProcess(processId);
             if (existing) {
               // Look for state id we don't know yet
-              let new_states = [];
+              let newStates: string[] = [];
-              let roles = [];
+              let newRoles: Record<string, RoleDefinition>[] = [];
               for (const state of process.states) {
-                if (!state.state_id || state.state_id === EMPTY32BYTES) { continue; }
+                if (!state || !state.state_id) { continue; } // shouldn't happen
-                if (!this.lookForStateId(existing, state.state_id)) {
+                if (state.state_id === EMPTY32BYTES) {
+                  // We check that the tip is the same we have, if not we update
+                  const existingTip = existing.states[existing.states.length - 1].commited_in;
+                  if (existingTip !== state.commited_in) {
+                    console.log('Found new tip for process', processId);
+                    existing.states.pop(); // We discard the last state
+                    existing.states.push(state);
+                    // We know that's the last state, so we just trigger the update
+                    toSave[processId] = existing;
+                  }
+                } else if (!this.lookForStateId(existing, state.state_id)) {
+                  // We don't want to overwrite what we already have for existing processes
+                  // We may end up overwriting the keys for example
+                  // So the process we're going to save needs to merge new states with what we already have
+                  const existingLastState = existing.states.pop();
+                  existing.states.push(state);
+                  existing.states.push(existingLastState);
+                  toSave[processId] = existing; // We mark it for update
                   if (this.rolesContainsUs(state.roles)) {
-                    new_states.push(state.state_id);
+                    newStates.push(state.state_id);
-                    roles.push(state.roles);
+                    newRoles.push(state.roles);
                   }
                 }
               }
 
-              if (new_states.length != 0) {
+              if (newStates.length != 0) {
-                // We request the new states
+                await this.requestDataFromPeers(processId, newStates, newRoles);
-                await this.requestDataFromPeers(processId, new_states, roles);
-                toSave[processId] = process;
               }
-
-              // Just to be sure check if that's a pairing process
-              const lastCommitedState = this.getLastCommitedState(process);
-              if (lastCommitedState && lastCommitedState.public_data && lastCommitedState.public_data['pairedAddresses']) {
-                // This is a pairing process
-                try {
-                  const pairedAddresses = this.decodeValue(lastCommitedState.public_data['pairedAddresses'] as unknown as number[]);
-                  // Are we part of it?
-                  if (pairedAddresses && pairedAddresses.length > 0 && pairedAddresses.includes(this.getDeviceAddress())) {
-                    // We save the process to db
-                    await this.saveProcessToDb(processId, process as Process);
-                    // We update the device
-                    await this.updateDevice();
-                  }
-                } catch (e) {
-                  console.error('Failed to check for pairing process:', e);
-                }
-              }
-
               // Otherwise we're probably just in the initial loading at page initialization
-
-              // We may learn an update for this process
-              // TODO maybe actually check if what the relay is sending us contains more information than what we have
-              // relay should always have more info than us, but we never know
-              // For now let's keep it simple and let the worker do the job
             } else {
               // We add it to db
-              console.log(`Saving ${processId} to db`);
               toSave[processId] = process;
             }
           }
 
-          await this.batchSaveProcessesToDb(toSave);
+          if (toSave && Object.keys(toSave).length > 0) {
+            console.log('batch saving processes to db', toSave);
+            await this.batchSaveProcessesToDb(toSave);
+          }
         }
       }, 500)
     } catch (e) {
@@ -264,8 +258,6 @@ export class Service {
     }));
   }
 
-
-
   /**
    * Get relay statistics from RelayManager.
    * @returns Statistics about connected relays
@@ -300,12 +292,53 @@ export class Service {
     }
   }
 
-  public async checkConnections(members: Member[]): Promise<void> {
+  // If we're updating a process, we must call that after update especially if roles are part of it
+  // We will take the roles from the last state, wheter it's commited or not
+  public async checkConnections(process: Process): Promise<void> {
+    const sharedSecret = await this.getAllSecretsFromDB();
+    console.log('sharedSecret found', sharedSecret);
+    if (process.states.length < 2) {
+      throw new Error('Process doesn\'t have any state yet');
+    }
+    let roles = process.states[process.states.length - 2].roles;
+    if (!roles) {
+      throw new Error('No roles found');
+    } else {
+      console.log('roles found', roles);
+    }
+    let members: Set<Member> = new Set();
+    for (const role of Object.values(roles!)) {
+      console.log('role found', role);
+      for (const member of role.members) {
+        console.log('member found', member);
+        // Check if we know the member that matches this id
+        const memberAddresses = this.getAddressesForMemberId(member);
+        console.log('memberAddresses found', memberAddresses);
+        if (memberAddresses && memberAddresses.length != 0) {
+          members.add({ sp_addresses: memberAddresses });
+        }
+      }
+    }
+
+    if (members.size === 0) {
+      // This must be a pairing process
+      // Check if we have a pairedAddresses in the public data
+      const publicData = process.states[0]?.public_data;
+      if (!publicData || !publicData['pairedAddresses']) {
+        throw new Error('Not a pairing process');
+      }
+      const decodedAddresses = this.decodeValue(publicData['pairedAddresses']);
+      if (decodedAddresses.length === 0) {
+        throw new Error('Not a pairing process');
+      }
+      members.add({ sp_addresses: decodedAddresses });
+    }
+
     // Ensure the amount is available before proceeding
     await this.getTokensFromFaucet();
-    let unconnectedAddresses = [];
+    let unconnectedAddresses = new Set<string>();
     const myAddress = this.getDeviceAddress();
-    for (const member of members) {
+    for (const member of Array.from(members)) {
       const sp_addresses = member.sp_addresses;
       if (!sp_addresses || sp_addresses.length === 0) continue;
       for (const address of sp_addresses) {
@@ -313,23 +346,23 @@ export class Service {
         if (address === myAddress) continue;
         const sharedSecret = await this.getSecretForAddress(address);
         if (!sharedSecret) {
-          unconnectedAddresses.push(address);
+          unconnectedAddresses.add(address);
         }
       }
     }
-    if (unconnectedAddresses && unconnectedAddresses.length != 0) {
+    if (unconnectedAddresses && unconnectedAddresses.size != 0) {
       const apiResult = await this.connectAddresses(unconnectedAddresses);
       await this.handleApiReturn(apiResult);
     }
   }
 
-  public async connectAddresses(addresses: string[]): Promise<ApiReturn> {
+  public async connectAddresses(addresses: Set<string>): Promise<ApiReturn> {
-    if (addresses.length === 0) {
+    if (addresses.size === 0) {
       throw new Error('Trying to connect to empty addresses list');
     }
 
     try {
-      return wasm.create_transaction(addresses, 1);
+      return wasm.create_transaction(Array.from(addresses), 1);
     } catch (e) {
       console.error('Failed to connect member:', e);
       throw e;
@@ -516,18 +549,6 @@ export class Service {
       ...wasm.encode_binary(publicSplitData.binaryData)
     };
 
-    let members: Set<Member> = new Set();
-    for (const role of Object.values(roles!)) {
-      for (const member of role.members) {
-        // Check if we know the member that matches this id
-        const memberAddresses = this.getAddressesForMemberId(member);
-        if (memberAddresses && memberAddresses.length != 0) {
-          members.add({ sp_addresses: memberAddresses });
-        }
-      }
-    }
-    await this.checkConnections([...members]);
-
     const result = wasm.create_new_process (
         encodedPrivateData, 
         roles,
@@ -536,8 +557,13 @@ export class Service {
         feeRate,
         this.getAllMembers()
     );
-
+    
-    return(result);
+    if (result.updated_process) {
+      await this.checkConnections(result.updated_process.current_process);
+      return result;
+    } else {
+      throw new Error('Failed to create new process');
+    }
   }
 
   async parseCipher(message: string): Promise<void> {
@@ -606,6 +632,7 @@ export class Service {
       }
 
       const result = wasm.create_update_message(process, stateId, this.membersList);  
+      await this.checkConnections(process);
       return result;
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : String(error || 'Unknown error');
@@ -624,7 +651,12 @@ export class Service {
       }
 
       const result = wasm.validate_state(process, stateId, this.membersList);
-      return result;
+      if (result.updated_process) {
+        await this.checkConnections(result.updated_process.current_process);
+        return result;
+      } else {
+        throw new Error('Failed to validate state');
+      }
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : String(error || 'Unknown error');
       throw new Error(errorMessage);
@@ -633,32 +665,46 @@ export class Service {
 
   // Core protocol method: Update Process
   async updateProcess(
-    process: any, 
+    process: Process, 
     privateData: Record<string, any>, 
     publicData: Record<string, any>, 
-    roles: Record<string, any> | null
+    roles: Record<string, RoleDefinition> | null
   ): Promise<ApiReturn> {
-    console.log(`🔄 Updating process ${process.states[0]?.state_id || 'unknown'}`);
+    console.log(`🔄 Updating process ${process.states[0]?.commited_in || 'unknown'}`);
     console.log('Private data:', privateData);
     console.log('Public data:', publicData);
-    console.log('Roles:', roles);
-    
-    try {
-      // Convert data to WASM format
-      const newAttributes = wasm.encode_json(privateData);
-      const newPublicData = wasm.encode_json(publicData);
-      const newRoles = roles || process.states[0]?.roles || {};
 
-      // Use WASM function to update process
+    if (!process || !process.states || process.states.length < 2) {
-      const result = wasm.update_process(process, newAttributes, newRoles, newPublicData, this.membersList);
+      throw new Error('Process not found');
+    }
+
+    if (!roles || Object.keys(roles).length === 0) {
+      const state = this.getLastCommitedState(process);
+      if (state) {
+        roles = state.roles;
+      } else {
+        roles = process.states[0]?.roles;
+      }
+    } else {
+      console.log('Roles provided:', roles);
+    }
+
+    const privateSplitData = this.splitData(privateData);
+    const publicSplitData = this.splitData(publicData);
+    const encodedPrivateData = { 
+      ...wasm.encode_json(privateSplitData.jsonCompatibleData), 
+      ...wasm.encode_binary(privateSplitData.binaryData)
+    };
+    const encodedPublicData = { 
+      ...wasm.encode_json(publicSplitData.jsonCompatibleData), 
+      ...wasm.encode_binary(publicSplitData.binaryData)
+    };
+
+    try {
+      const result = wasm.update_process(process, encodedPrivateData, roles, encodedPublicData, this.membersList);
       
       if (result.updated_process) {
-        // Update our cache
+        await this.checkConnections(result.updated_process.current_process); 
-        this.processes.set(result.updated_process.process_id, result.updated_process.current_process);
-        
-        // Save to database
-        await this.saveProcessToDb(result.updated_process.process_id, result.updated_process.current_process);
-        
         return result;
       } else {
         throw new Error('Failed to update process');
@@ -685,7 +731,7 @@ export class Service {
       const newMyProcesses = new Set<string>();
       // MyProcesses automatically contains pairing process
       newMyProcesses.add(pairingProcessId);
-      for (const [processId, process] of Object.entries(this.processes)) {
+      for (const [processId, process] of this.processes.entries()) {
         try {
           const roles = this.getRoles(process);
 
@@ -703,6 +749,47 @@ export class Service {
     }
   }
 
+  public async getProcessesData(filteredProcesses: Record<string, Process>): Promise<Record<string, any>> {
+    const data: Record<string, any> = {};
+    // Now we decrypt all we can in the processes
+    for (const [processId, process] of Object.entries(filteredProcesses)) {
+      // We also take the public data
+      let lastState = this.getLastCommitedState(process);
+      if (!lastState) {
+        // fallback on the first state
+        lastState = process.states[0];
+      }
+      const processData: Record<string, any> = {};
+      for (const attribute of Object.keys(lastState.public_data)) {
+        try {
+          const value = this.decodeValue(lastState.public_data[attribute]);
+          if (value !== null && value !== undefined) {
+            processData[attribute] = value;
+          }
+        } catch (e) {
+          console.error(`❌ Error decoding public data ${attribute} for process ${processId}:`, e);
+        }
+      }
+      for (let i = process.states.length - 2; i >= 0; i--) {
+        const state = process.states[i];
+        for (const attribute of Object.keys(state.keys)) {
+          if (processData[attribute] !== undefined && processData[attribute] !== null) continue;
+          try {
+            const value = await this.decryptAttribute(processId, state, attribute);
+            if (value) {
+              processData[attribute] = value;
+            }
+          } catch (e) {
+            console.error(`❌ Error decrypting attribute ${attribute} for process ${processId}:`, e);
+          }
+        }
+      }
+      data[processId] = processData;
+    }
+
+    return data;
+  }
+
   // Utility method: Get Process
   async getProcess(processId: string): Promise<any | null> {
     // First check in-memory cache
@@ -767,6 +854,25 @@ export class Service {
     }
   }
 
+  public async getAllSecretsFromDB(): Promise<SecretsStore> {
+    try {
+      const db = await Database.getInstance();
+      const sharedSecrets: Record<string, string> = await db.dumpStore('shared_secrets');
+      const unconfirmedSecrets = await db.dumpStore('unconfirmed_secrets');
+      const secretsStore = {
+        shared_secrets: sharedSecrets,
+        unconfirmed_secrets: Object.values(unconfirmedSecrets),
+      };
+      return secretsStore;
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  public loadSecretsInWasm(secretsStore: SecretsStore) {
+    wasm.set_shared_secrets(JSON.stringify(secretsStore));
+  }
+
   // Utility method: Create a test process
   async createTestProcess(processId: string): Promise<any> {
     console.log(`🔧 Creating test process: ${processId}`);
@@ -952,37 +1058,35 @@ export class Service {
   }
 
   // Blob and data storage methods
-  async saveBlobToDb(hash: string, data: Blob) {
+  async saveBufferToDb(hash: string, data: Buffer) {
     const db = await Database.getInstance();
     try {
       await db.addObject({
         storeName: 'data',
         object: data,
         key: hash,
-      });
+      }, true);
     } catch (e) {
       console.error(`Failed to save data to db: ${e}`);
     }
   } 
 
-  async getBlobFromDb(hash: string): Promise<Blob | null> {
+  async getBufferFromDb(hash: string): Promise<Buffer | null> {
     const db = await Database.getInstance();
     try {
-      return await db.getObject('data', hash);
+      return await db.getObject('data', hash, true);
     } catch (e) {
       return null;
     }
   }
 
-  async saveDataToStorage(hash: string, data: Blob, ttl: number | null) {
+  async saveDataToStorage(hash: string, data: Buffer, ttl: number | null, storageUrls: string[]) {
-    console.log('💾 Saving data to storage:', hash);
+    console.log('💾 Saving data', hash, 'to storage', storageUrls);
-    // TODO: Implement actual storage service
+    try {
-    // const storages = [STORAGEURL];
+      await storeData(storageUrls, hash, data, ttl);
-    // try {
+    } catch (e) {
-    //   await storeData(storages, hash, data, ttl);
+      console.error(`Failed to store data with hash ${hash}: ${e}`);
-    // } catch (e) {
+    }
-    //   console.error(`Failed to store data with hash ${hash}: ${e}`);
-    // }
   }
 
   async saveDiffsToDb(diffs: any[]) {
@@ -1000,6 +1104,11 @@ export class Service {
     }
   }
 
+  async getDiffsFromDb(): Promise<Record<string, UserDiff>> {
+    const db = await Database.getInstance();
+    return await db.dumpStore('diffs');
+  }
+
   // Utility methods for data conversion
   hexToBlob(hexString: string): Blob {
     const uint8Array = this.hexToUInt8Array(hexString);
@@ -1017,6 +1126,10 @@ export class Service {
     return uint8Array;
   }
 
+  hexToBuffer(hexString: string): Buffer {
+    return Buffer.from(this.hexToUInt8Array(hexString));
+  }
+
   public async handleApiReturn(apiReturn: ApiReturn) {
     // Check for errors in the returned objects
     if (apiReturn.new_tx_to_send && apiReturn.new_tx_to_send.error) {
@@ -1081,9 +1194,9 @@ export class Service {
 
       if (updatedProcess.encrypted_data && Object.keys(updatedProcess.encrypted_data).length != 0) {
         for (const [hash, cipher] of Object.entries(updatedProcess.encrypted_data)) {
-          const blob = this.hexToBlob(cipher);
+          const buffer = this.hexToBuffer(cipher);
           try {
-            await this.saveBlobToDb(hash, blob);
+            await this.saveBufferToDb(hash, buffer);
           } catch (e) {
             console.error(e);
           }
@@ -1104,11 +1217,27 @@ export class Service {
 
     if (apiReturn.push_to_storage && apiReturn.push_to_storage.length != 0) {
       for (const hash of apiReturn.push_to_storage) {
-        const blob = await this.getBlobFromDb(hash);
+        const buffer = await this.getBufferFromDb(hash);
-        if (blob) {
+        if (buffer) {
-          await this.saveDataToStorage(hash, blob, null);
+          // Look up the storage url for the hash
+          // Find the field for this hash, then look up the roles to see what storage urls are associated
+          let storageUrls = new Set<string>();
+          const diffs = await this.getDiffsFromDb();
+          const diff = Object.values(diffs).find((diff: UserDiff) => diff.value_commitment === hash);
+          if (diff) {
+            for (const role of Object.values(diff.roles)) {
+              for (const rule of Object.values(role.validation_rules)) {
+                if (rule.fields.includes(diff.field)) {
+                  for (const storageUrl of role.storages) {
+                    storageUrls.add(storageUrl);
+                  }
+                }
+              }
+            }
+          }
+          await this.saveDataToStorage(hash, buffer, null, Array.from(storageUrls));
         } else {
-          console.error('Failed to get data from db');
+          console.error('Failed to get data from db for hash:', hash);
         }
       }
     }
@@ -1207,11 +1336,10 @@ export class Service {
     }
 
     if (hash && key) {
-      const blob = await this.getBlobFromDb(hash);
+      const buffer = await this.getBufferFromDb(hash);
-      if (blob) {
+      if (buffer) {
         // Decrypt the data
-        const buf = await blob.arrayBuffer();
+        const cipher = new Uint8Array(buffer);
-        const cipher = new Uint8Array(buf);
 
         const keyUIntArray = this.hexToUInt8Array(key);
 
@@ -1220,7 +1348,7 @@ export class Service {
           if (clear) {
             // deserialize the result to get the actual data
             const decoded = wasm.decode_value(clear);
-            return decoded;
+            return this.convertMapsToObjects(decoded);
           } else {
             throw new Error('decrypt_data returned null');
           }
@@ -1235,13 +1363,49 @@ export class Service {
 
   decodeValue(value: number[]): any | null {
     try {
-      return wasm.decode_value(new Uint8Array(value));
+      const decoded = wasm.decode_value(new Uint8Array(value));
+      return this.convertMapsToObjects(decoded);
     } catch (e) {
       console.error(`Failed to decode value: ${e}`);
       return null;
     }
   }
 
+  /**
+   * Convertit récursivement les Map en objets sérialisables
+   */
+  private convertMapsToObjects(obj: any): any {
+    if (obj === null || obj === undefined) {
+      return obj;
+    }
+
+    if (obj instanceof Map) {
+      const result: any = {};
+      for (const [key, value] of obj.entries()) {
+        result[key] = this.convertMapsToObjects(value);
+      }
+      return result;
+    }
+
+    if (obj instanceof Set) {
+      return Array.from(obj).map(item => this.convertMapsToObjects(item));
+    }
+
+    if (Array.isArray(obj)) {
+      return obj.map(item => this.convertMapsToObjects(item));
+    }
+
+    if (typeof obj === 'object') {
+      const result: any = {};
+      for (const [key, value] of Object.entries(obj)) {
+        result[key] = this.convertMapsToObjects(value);
+      }
+      return result;
+    }
+
+    return obj;
+  }
+
   public async updateDevice(): Promise<void> {
     let myPairingProcessId: string;
     try {

src/simple-server.ts

@@ -3,7 +3,7 @@ import { MessageType } from './models';
 import { config } from './config';
 import { Service } from './service';
 import { ApiReturn, Process } from '../pkg/sdk_client';
-import { EMPTY32BYTES } from './utils';
+import { EMPTY32BYTES, splitPrivateData } from './utils';
 
 interface ServerMessageEvent {
   data: {
@@ -42,6 +42,52 @@ class SimpleProcessHandlers {
     return apiKey === this.apiKey;
   }
 
+  async handleCreateProcess(event: ServerMessageEvent): Promise<ServerResponse> {
+    if (event.data.type !== MessageType.CREATE_PROCESS) {
+      throw new Error('Invalid message type');
+    }
+
+    if (!this.service.isPaired()) {
+      throw new Error('Device not paired');
+    }
+
+    try {
+      const { processData, privateFields, roles, exclusionRules, apiKey } = event.data;
+
+      if (!apiKey || !this.validateApiKey(apiKey)) {
+        throw new Error('Invalid API key');
+      }
+
+      const { privateData, publicData } = splitPrivateData(processData, privateFields);
+
+      const createProcessReturn = await this.service.createProcess(privateData, publicData, roles);
+      if (!createProcessReturn.updated_process) {
+        throw new Error('Empty updated_process in createProcessReturn');
+      }
+      console.log('🚀 ~ handleCreateProcess ~ createProcessReturn:', createProcessReturn);
+      const processId = createProcessReturn.updated_process.process_id;
+      const process = createProcessReturn.updated_process.current_process;
+      await this.service.handleApiReturn(createProcessReturn);
+
+      const processCreated = {
+        processId,
+        process,
+        processData,
+      }
+
+      return {
+        type: MessageType.PROCESS_CREATED,
+        processCreated,
+        messageId: event.data.messageId
+      };
+    } catch (e) {
+      const errorMessage = e instanceof Error ? e.message : String(e || 'Unknown error');
+      // Remove redundant "Error:" prefix and simplify the message
+      const cleanMessage = errorMessage.replace(/^Error:\s*/, '');
+      throw new Error(cleanMessage);
+    }
+  }
+
   async handleNotifyUpdate(event: ServerMessageEvent): Promise<ServerResponse> {
     if (event.data.type !== MessageType.NOTIFY_UPDATE) {
       throw new Error('Invalid message type');
@@ -216,71 +262,70 @@ class SimpleProcessHandlers {
       throw new Error('Invalid message type');
     }
     
-    const processes = this.service.getProcesses();
+    if (!this.service.isPaired()) {
-    const myProcesses = await this.service.getMyProcesses();
+      throw new Error('Device not paired');
-
-    if (!myProcesses || myProcesses.length === 0) {
-      throw new Error('No my processes found');
     }
 
-    const filteredProcesses: Record<string, Process> = {};
+    try {
-    for (const processId of myProcesses) {
+      const processes = this.service.getProcesses();
-      const process = processes.get(processId);
+      const myProcesses = await this.service.getMyProcesses();
-      console.log(processId, ':', process);
 
-      if (process) {
+      if (!myProcesses || myProcesses.length === 0) {
-        filteredProcesses[processId] = process;
+        throw new Error('No my processes found');
       }
-    }
 
-    const data: Record<string, any> = {};
+      const filteredProcesses: Record<string, Process> = {};
-    // Now we decrypt all we can in the processes
+      for (const processId of myProcesses) {
-    for (const [processId, process] of Object.entries(filteredProcesses)) {
+        const process = processes.get(processId);
-      // We also take the public data
+
-      const lastState = this.service.getLastCommitedState(process);
+        if (process) {
-      if (!lastState) {
+          filteredProcesses[processId] = process;
-        console.error(`❌ Process ${processId} doesn't have a commited state`);
+        } else {
-        continue;
+          console.error(`Process ${processId} not found`); // should not happen
-      }
-      const processData: Record<string, any> = {};
-      for (const attribute of Object.keys(lastState.public_data)) {
-        try {
-          const value = this.service.decodeValue(lastState.public_data[attribute]);
-          if (value) {
-            processData[attribute] = value;
-          }
-        } catch (e) {
-          console.error(`❌ Error decoding public data ${attribute} for process ${processId}:`, e);
         }
       }
-      for (let i = process.states.length - 2; i >= 0; i--) {
+
-        const state = process.states[i];
+      const data = await this.service.getProcessesData(filteredProcesses);
-        for (const attribute of Object.keys(state.keys)) {
+
-          if (processData[attribute] !== undefined && processData[attribute] !== null) continue;
+      return {
-          try {
+        type: MessageType.PROCESSES_RETRIEVED,
-            const value = await this.service.decryptAttribute(processId, state, attribute);
+        processes: filteredProcesses,
-            if (value) {
+        data,
-              processData[attribute] = value;
+        messageId: event.data.messageId
-            }
+      };      
-          } catch (e) {
+    } catch (e) {
-            console.error(`❌ Error decrypting attribute ${attribute} for process ${processId}:`, e);
+      const errorMessage = e instanceof Error ? e.message : String(e || 'Unknown error');
-          }
+      throw new Error(errorMessage);
-        }
+    }
-      }
+  }
-      data[processId] = processData;
+
+  async handleGetPairingId(event: ServerMessageEvent): Promise<ServerResponse> {
+    if (event.data.type !== MessageType.GET_PAIRING_ID) {
+      throw new Error('Invalid message type');
     }
 
-    return {
+    if (!this.service.isPaired()) {
-      type: MessageType.PROCESSES_RETRIEVED,
+      throw new Error('Device not paired');
-      processes: filteredProcesses,
+    }
-      data,
+
-      messageId: event.data.messageId
+    try {
-    };
+      const pairingId = this.service.getPairingProcessId();
+      return {
+        type: MessageType.GET_PAIRING_ID,
+        pairingId,
+        messageId: event.data.messageId
+      };
+    } catch (e) {
+      const errorMessage = e instanceof Error ? e.message : String(e || 'Unknown error');
+      throw new Error(errorMessage);
+    }
   }
 
   async handleMessage(event: ServerMessageEvent): Promise<ServerResponse> {
     try {
       switch (event.data.type) {
+        case MessageType.CREATE_PROCESS:
+          return await this.handleCreateProcess(event);
         case MessageType.NOTIFY_UPDATE:
           return await this.handleNotifyUpdate(event);
         case MessageType.VALIDATE_STATE:
@@ -289,6 +334,8 @@ class SimpleProcessHandlers {
           return await this.handleUpdateProcess(event);
         case MessageType.GET_MY_PROCESSES:
           return await this.handleGetMyProcesses(event);
+        case MessageType.GET_PAIRING_ID:
+          return await this.handleGetPairingId(event);
         default:
           throw new Error(`Unhandled message type: ${event.data.type}`);
       }
@@ -353,15 +400,14 @@ export class Server {
           if (!processId || !stateId) {
             throw new Error('Failed to get process id or state id');
           }
+          // now pair the device before continuing
+          service.pairDevice(processId, [service.getDeviceAddress()]);
           await service.handleApiReturn(pairingResult);
           const udpateResult = await service.createPrdUpdate(processId, stateId);
           await service.handleApiReturn(udpateResult);
           const approveResult = await service.approveChange(processId, stateId);
           await service.handleApiReturn(approveResult);
 
-          // now pair the device
-          service.pairDevice(processId, [service.getDeviceAddress()]);
-
           // Update the device in the database
           const device = service.dumpDeviceFromMemory();
           if (device) {
@@ -380,6 +426,9 @@ export class Server {
 
       // Get all processes from database
       await service.getAllProcessesFromDb();
+      const secretsStore = await service.getAllSecretsFromDB();
+
+      service.loadSecretsInWasm(secretsStore);
 
       // Connect to relays
       await service.connectToRelaysAndWaitForHandshake();

src/storage.service.ts

@@ -0,0 +1,111 @@
+import axios, { AxiosResponse } from 'axios';
+
+export async function storeData(servers: string[], key: string, value: Buffer, ttl: number | null): Promise<AxiosResponse | null> {
+  for (const server of servers) {
+    try {
+      // Use key in the URL path instead of query parameters
+      let url = `${server}/store/${key}`;
+      
+      // Add ttl as query parameter if provided
+      if (ttl !== null) {
+        const urlObj = new URL(url);
+        urlObj.searchParams.append('ttl', ttl.toString());
+        url = urlObj.toString();
+      }
+
+      // Send the encrypted ArrayBuffer as the raw request body.
+      const response = await axios.post(url, value, {
+        headers: {
+          'Content-Type': 'application/octet-stream'
+        },
+      });
+      console.log('Data stored successfully:', key);
+      if (response.status !== 200) {
+        console.error('Received response status', response.status);
+        continue;
+      }
+      return response;
+    } catch (error) {
+      if (axios.isAxiosError(error) && error.response?.status === 409) {
+        return null;
+      }
+      console.error('Error storing data:', error);
+    }
+  }
+  return null;
+}
+
+export async function retrieveData(servers: string[], key: string): Promise<ArrayBuffer | null> {
+  for (const server of servers) {
+    try {
+      // Handle relative paths (for development proxy) vs absolute URLs (for production)
+      const url = server.startsWith('/') 
+        ? `${server}/retrieve/${key}`  // Relative path - use as-is for proxy
+        : new URL(`${server}/retrieve/${key}`).toString();  // Absolute URL - construct properly
+      
+      console.log('Retrieving data', key,' from:', url);
+      // When fetching the data from the server:
+      const response = await axios.get(url, {
+        responseType: 'arraybuffer' 
+      });
+      
+      if (response.status === 200) {
+        // Validate that we received an ArrayBuffer
+        if (response.data instanceof ArrayBuffer) {
+          return response.data;
+        } else {
+          console.error('Server returned non-ArrayBuffer data:', typeof response.data);
+          continue;
+        }
+      } else {
+        console.error(`Server ${server} returned status ${response.status}`);
+        continue;
+      }
+    } catch (error) {
+      if (axios.isAxiosError(error)) {
+        if (error.response?.status === 404) {
+          console.log(`Data not found on server ${server} for key ${key}`);
+          continue; // Try next server
+        } else if (error.response?.status) {
+          console.error(`Server ${server} error ${error.response.status}:`, error.response.statusText);
+          continue;
+        } else {
+          console.error(`Network error connecting to ${server}:`, error.message);
+          continue;
+        }
+      } else {
+        console.error(`Unexpected error retrieving data from ${server}:`, error);
+        continue;
+      }
+    }
+  }
+  return null;
+}
+
+interface TestResponse {
+    key: string;
+    value: boolean;
+}
+
+export async function testData(servers: string[], key: string): Promise<Record<string, boolean | null> | null> {
+    const res: Record<string, boolean | null> = {};
+    for (const server of servers) {
+        res[server] = null;
+        try {
+            const response = await axios.get(`${server}/test/${key}`);
+            if (response.status !== 200) {
+                console.error(`${server}: Test response status: ${response.status}`);
+                continue;
+            }
+            
+            const data: TestResponse = response.data;
+            
+            res[server] = data.value;
+        } catch (error) {
+            console.error('Error retrieving data:', error);
+            return null;
+        }
+    }
+    
+    return res;
+}

src/utils.ts

@@ -1,6 +1,21 @@
 // Server-specific utility functions
 export const EMPTY32BYTES = String('').padStart(64, '0');
 
+export function splitPrivateData(data: Record<string, any>, privateFields: string[]): { privateData: Record<string, any>, publicData: Record<string, any> } {
+  const privateData: Record<string, any> = {};
+  const publicData: Record<string, any> = {};
+
+  for (const [key, value] of Object.entries(data)) {
+    if (privateFields.includes(key)) {
+      privateData[key] = value;
+    } else {
+      publicData[key] = value;
+    }
+  }
+
+  return { privateData, publicData };
+}
+
 export function isValid32ByteHex(value: string): boolean {
   // Check if the value is a valid 32-byte hex string (64 characters)
   const hexRegex = /^[0-9a-fA-F]{64}$/;