ident

19.9 KB

1. // tests/realdata-baseline.mjs (ident mission 009, code order): every read ident answers + a fixed write sequence, on a
2. // COPY of the LIVE storage, saved for diffing two code trees (a cleanup must answer the same). Compare two outputs with
3. // tests/realdata-compare.py. README "Test" → "Same output". Copied from gitoria's (mission 002) and made for ident.
4. // node tests/realdata-baseline.mjs <tree> <port> <outdir> (kills only the server it started)
5. // <tree> must be a CODE-ONLY copy (no .env: the server reads .env from its cwd; the mail goes to a sink file anyway).
6. // The live copy: IDENT_REALDATA (default .scratch/m009/realdata) must hold mpackdb/ (Byrodin's storage/mpackdb), e.g.
7. // tar -C /CONTAINERS/projects/ident.worldapi.org/storage -cf - mpackdb | ssh loreana tar -C <repo>/.scratch/m009/realdata -xf -
8. // Each run works on a fresh copy of it (<outdir>.run, removed after). Signed in as the creator (identity az5b2 → account
9. // 0mufd5afy0xm) through a session file written before the start; every page is fetched signed in AND signed out.
10. // Writes: identities, time zone, an app (key/secret), the login button + exchange, the selector, kinds + notifications,
11. // inbox switches, invites, migrate-ids, the OTP login of a new and of the creator's address (mail sink), sign out
12. // (everywhere), app delete — then the stored state (tools/dump-store.hl on a copy).
13. import { spawn, execSync, execFileSync } from 'node:child_process';
14. import { request } from 'node:http';
15. import { writeFileSync, mkdirSync, rmSync, existsSync, readFileSync } from 'node:fs';
16. import { createHash, randomBytes } from 'node:crypto';
17. import { join, resolve } from 'node:path';
19. const [tree, portArg, outArg] = process.argv.slice(2);
20. const PORT = Number(portArg);
21. const W = resolve(process.env.IDENT_REALDATA || '.scratch/m009/realdata'), OUT = resolve(outArg), RUN = OUT + '.run';
22. const ACCOUNT = '0mufd5afy0xm', IDENTITY = '0mufd5afyezw', SHORT = 'az5b2';
23. if (existsSync(join(resolve(tree), '.env'))) throw new Error('the tree has a .env — use a code-only copy');
24. rmSync(RUN, { recursive: true, force: true }); rmSync(OUT, { recursive: true, force: true });
25. mkdirSync(join(RUN, 'sessions'), { recursive: true }); mkdirSync(OUT, { recursive: true });
26. execSync(`cp -a ${W}/mpackdb ${RUN}/mpackdb`);
27. const SINK = join(RUN, 'mail.txt');
29. // the creator's address, read off a dump of a copy (opening a table rewrites it, hybriel #40)
30. const dumpOf = (src) => {
31. const cp = RUN + '/dump-copy'; rmSync(cp, { recursive: true, force: true }); execSync(`cp -a ${src} ${cp}`);
32. const out = execFileSync(resolve(tree, 'bin/hybriel'), ['tools/dump-store.hl'], { cwd: resolve(tree), env: { ...process.env, IDENT_STORAGE: cp, IDENT_WATCH: '0' }, encoding: 'utf8' });
33. rmSync(cp, { recursive: true, force: true });
34. return JSON.parse(out.slice(out.indexOf('{')));
35. };
36. const before = dumpOf(join(RUN, 'mpackdb'));
37. const EMAIL = before.accounts.find(a => a.id === ACCOUNT).email;
38. const conns = before.connections.map(c => c.id).sort();
39. const apps = before.apps.slice().sort((a, b) => a.name < b.name ? -1 : 1);
41. // ---- the session (hl:web: file name sha256(sid), JSON) -------------------------------------------------------------
42. const sid = randomBytes(16).toString('hex'), now = Date.now();
43. writeFileSync(join(RUN, 'sessions', createHash('sha256').update(sid).digest('hex')),
44. JSON.stringify({ created: now, data: {}, id: sid, seen: now, user: { id: ACCOUNT } }), { mode: 0o600 });
45. const COOKIE = 'identsid=' + sid;
46. const sleep = ms => new Promise(r => setTimeout(r, ms));
47. const HOST = `ident.test:${PORT}`;
49. const srv = spawn(resolve(tree, 'bin/hybriel'), ['project.hl'], { cwd: resolve(tree), stdio: ['ignore', 'pipe', 'pipe'],
50. env: { ...process.env, IDENT_PORT: String(PORT), HL_HOST: '127.0.0.1', IDENT_STORAGE: join(RUN, 'mpackdb'), IDENT_SESSIONS: join(RUN, 'sessions') + '/',
51. IDENT_MAIL_SINK: SINK, IDENT_WATCH: '0', IDENT_IP_LIMIT: '1000', IDENT_IP_DAY_LIMIT: '1000', IDENT_PUBLIC_URL: '',
52. SMTP_HOST: '', SMTP_USER: '', SMTP_PASSWORD: '' } });
53. let log = ''; srv.stdout.on('data', d => log += d); srv.stderr.on('data', d => log += d);
55. const hreq = (method, path, headers = {}, body = null) => new Promise((res) => {
56. const rq = request({ host: '127.0.0.1', port: PORT, path, method, headers: { host: HOST, ...headers } }, (r) => {
57. const parts = []; r.on('data', d => parts.push(d)); r.on('end', () => res({ status: r.statusCode, headers: r.headers, body: Buffer.concat(parts).toString('latin1') }));
58. });
59. rq.on('error', (e) => res({ status: 0, headers: {}, body: 'ERROR ' + e.message }));
60. if (body != null) { rq.setHeader('content-length', Buffer.byteLength(body)); rq.write(body); }
61. rq.end();
62. });
63. let n = 0;
64. const file = (name, text) => writeFileSync(join(OUT, String(++n).padStart(4, '0') + '-' + name.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 120)), text);
65. const head = (r) => [r.status, r.headers['content-type'] || '', r.headers['location'] || '', r.headers['cache-control'] || '',
66. r.headers['access-control-allow-origin'] || '', r.headers['access-control-allow-credentials'] || '', r.headers['access-control-allow-methods'] || '',
67. r.headers['vary'] || '', r.headers['set-cookie'] ? 'set-cookie' : ''].join(' | ');
68. const save = async (name, method, path, headers = {}, body = null) => {
69. const r = await hreq(method, path, headers, body);
70. file(name, head(r) + '\n' + r.body);
71. return r;
72. };
73. const J = JSON.stringify;
74. const post = (name, path, body, headers = {}) => save(name, 'POST', path, { 'content-type': 'application/json', ...headers }, typeof body === 'string' ? body : J(body));
75. const val = (r) => { try { return JSON.parse(r.body); } catch { return null; } };
76. let ei = 0;
77. // a face over hl:web's POST carrier; answers { value, cookie (a fresh session's) }
78. const face = async (name, event, payload, cookie = COOKIE) => {
79. const r = await hreq('POST', '/__hl/emit', { 'content-type': 'application/json', ...(cookie ? { cookie } : {}) }, J({ t: 'emit', i: ++ei, event, payload }));
80. file('w-face-' + name, head(r) + '\n' + r.body);
81. let v = null; try { v = JSON.parse(r.body).value; } catch {}
82. return { value: v, cookie: r.headers['set-cookie'] ? r.headers['set-cookie'][0].split(';')[0] : null };
83. };
84. const lastCode = (email) => {
85. const lines = readFileSync(SINK, 'utf8').trim().split('\n').filter(l => l.startsWith(email + ' '));
86. return lines.length ? lines[lines.length - 1].split(' ')[1] : null;
87. };
89. try {
90. for (let i = 0; i < 240; i++) { if ((await hreq('GET', '/api/online')).status === 204) break; await sleep(250); }
91. // ---- READS (signed in and signed out) ------------------------------------------------------------------------------
92. const pages = ['/', '/start', '/apps', '/inbox', '/inbox/nosuch', '/inbox/' + 'x'.repeat(200), ...conns.map(c => '/inbox/' + c),
93. '/code', '/signin/' + 'f'.repeat(32), '/signin/' + 'f'.repeat(32) + '/code', '/signin/xyz/code', '/signin/xyz', '/nope', '/nope/deeper',
94. '/selector.js', '/avatar.js', '/favicon.ico', '/icons/icon.svg', '/icons/icon-192.png', '/api/online', '/api/nope', '/api/exchange',
95. '/api/code', '/api/kinds', '/api/notify', '/api/invites', '/api/invites/list', '/api/invites/get', '/api/invites/revoke', '/api/migrate-ids',
96. '/login', '/login?key=pk_nope&return=https://x.example', '/invite/nosuch', '/invite/' + 'a'.repeat(100),
97. '/__hl/app.css', '/__hl/manifest.webmanifest', '/__hl/sw.js'];
98. for (const a of apps) {
99. pages.push('/login?key=' + a.apiKey, '/login?key=' + a.apiKey + '&return=https://evil.example/x', '/login?key=' + a.apiKey + '&return=' + encodeURIComponent('javascript:alert(1)'),
100. '/login?key=' + a.apiKey + '&return=' + encodeURIComponent(a.origins.split(' ')[0] + '@evil.example/'));
101. }
102. const modules = new Set();
103. for (const p of pages) {
104. const r = await save('in-' + p, 'GET', p, { cookie: COOKIE }); await save('out-' + p, 'GET', p);
105. for (const m of r.body.matchAll(/(\/(?:components|__hl)\/[\w./-]+\.(?:hl|js|css))\?v=[0-9a-f]+/g)) modules.add(m[0]);
106. }
107. for (const m of [...modules].sort()) await save('module-' + m.replace(/\?v=.*/, ''), 'GET', m);
108. // the selector of every live app: its origin signed in / out, a foreign origin, no origin, the preflight, a bad choose
109. for (const a of apps) {
110. const o = a.origins.split(' ')[0];
111. await save('sel-in-' + a.name, 'GET', '/api/selector/identities?key=' + a.apiKey, { origin: o, cookie: COOKIE });
112. await save('sel-out-' + a.name, 'GET', '/api/selector/identities?key=' + a.apiKey, { origin: o });
113. await save('sel-evil-' + a.name, 'GET', '/api/selector/identities?key=' + a.apiKey, { origin: 'https://evil.example', cookie: COOKIE });
114. await save('sel-pre-' + a.name, 'OPTIONS', '/api/selector/choose?key=' + a.apiKey, { origin: o });
115. await save('sel-bad-' + a.name, 'POST', '/api/selector/choose?key=' + a.apiKey, { origin: o, cookie: COOKIE, 'content-type': 'application/json' }, J({ identity: 'nope' }));
116. await save('sel-post-ids-' + a.name, 'POST', '/api/selector/identities?key=' + a.apiKey, { origin: o, cookie: COOKIE });
117. }
118. await save('sel-noorigin', 'GET', '/api/selector/identities?key=' + apps[0].apiKey, { cookie: COOKIE });
119. await save('sel-nokey', 'GET', '/api/selector/identities', { origin: 'https://tickets.worldapi.org', cookie: COOKIE });
120. // the app APIs refused (no secret is known for a live app)
121. for (const [p, body] of [['/api/exchange', { key: apps[0].apiKey, secret: 'sk_x', code: 'c' }], ['/api/exchange', { key: apps[0].apiKey, secret: ' ', code: 'c' }],
122. ['/api/exchange', '{"key":'], ['/api/exchange', '[1]'], ['/api/exchange', { key: 1, secret: 's', code: 'c' }], ['/api/exchange', { key: 'k', secret: 's', code: 'c', x: 1 }],
123. ['/api/kinds', { key: apps[0].apiKey, secret: 'sk_x', kinds: [] }], ['/api/notify', { key: apps[0].apiKey, secret: 'sk_x', identity: SHORT, name: 'n', text: 't' }],
124. ['/api/invites', { key: apps[0].apiKey, secret: 'sk_x', project: 'p', role: 'r', return: 'https://tickets.worldapi.org/' }], ['/api/invites/list', { key: 'k', secret: 's' }],
125. ['/api/invites/get', { key: 'k', secret: 's', id: 'x' }], ['/api/invites/revoke', { key: 'k', secret: 's' }], ['/api/migrate-ids', { key: apps[0].apiKey, secret: 'sk_x' }],
126. ['/api/code', { email: 'not-an-address' }], ['/api/code', { email: 5 }], ['/api/code', ''], ['/api/code', { email: '[email protected]', more: 1 }]]) {
127. await post('api-refused-' + p, p, body);
128. }
130. // ---- WRITES ---------------------------------------------------------------------------------------------------------
131. // identities + time zone (the creator, through the session file)
132. let f = await face('addIdentity', 'addIdentity', [{ identityName: 'RD', nickname: 'rd', firstname: 'Real', lastname: 'Data' }]);
133. const newIdent = f.value && f.value.identity;
134. await face('addIdentity-bad', 'addIdentity', [{ nope: 'x' }]);
135. await face('editIdentity', 'editIdentity', [newIdent, { nickname: 'rd2', avatar: '' }]);
136. await face('editIdentity-other', 'editIdentity', ['0mufd5afy623', { nickname: 'x' }]);
137. await face('changeTimeZone-bad', 'changeTimeZone', ['Not a zone!']);
138. await face('changeTimeZone', 'changeTimeZone', ['Europe/Berlin']);
139. await save('w-page-identities', 'GET', '/', { cookie: COOKIE });
140. await face('dropIdentity', 'dropIdentity', [newIdent]);
141. await face('noSession-addIdentity', 'addIdentity', [{}], null);
142. // an app
143. f = await face('appCreate-bad', 'appCreate', [{ name: 'RD', origins: ['ftp://x'] }]);
144. f = await face('appCreate', 'appCreate', [{ name: 'RD app', origins: ['https://rd.example.org', 'http://127.0.0.1:9'] }]);
145. const app = f.value.app; let secret = f.value.secret;
146. await face('appUpdate', 'appUpdate', [app.id, { name: 'RD app 2', origins: ['https://rd.example.org'] }]);
147. f = await face('appNewSecret', 'appNewSecret', [app.id]);
148. const oldSecret = secret; secret = f.value.secret;
149. await save('w-page-apps', 'GET', '/apps', { cookie: COOKIE });
150. const K = app.apiKey, ORIGIN = 'https://rd.example.org';
151. await post('w-kinds-oldsecret', '/api/kinds', { key: K, secret: oldSecret, kinds: [] });
152. await post('w-kinds', '/api/kinds', { key: K, secret, kinds: [{ name: 'comment', push: true, email: false }, { name: 'mention', push: false, email: true }] });
153. await post('w-kinds-bad', '/api/kinds', { key: K, secret, kinds: [{ name: 'x', push: 1, email: true }] });
154. // the login button
155. let r = await save('w-login', 'GET', '/login?key=' + K + '&return=' + encodeURIComponent(ORIGIN + '/back?x=1'));
156. let rid = r.headers.location.split('/').pop();
157. await save('w-signin-in', 'GET', '/signin/' + rid, { cookie: COOKIE });
158. await save('w-signin-out', 'GET', '/signin/' + rid);
159. await save('w-signin-code', 'GET', '/signin/' + rid + '/code', { cookie: COOKIE });
160. f = await face('chooseIdentity', 'chooseIdentity', [rid, IDENTITY]);
161. const code = new URL(f.value.url).searchParams.get('ident_code');
162. await face('chooseIdentity-again', 'chooseIdentity', [rid, IDENTITY]);
163. await post('w-exchange', '/api/exchange', { key: K, secret, code });
164. await post('w-exchange-again', '/api/exchange', { key: K, secret, code });
165. // the selector
166. r = await save('w-sel-ids', 'GET', '/api/selector/identities?key=' + K, { origin: ORIGIN, cookie: COOKIE });
167. const pick = val(r).identities[0].id;
168. r = await save('w-sel-choose', 'POST', '/api/selector/choose?key=' + K, { origin: ORIGIN, cookie: COOKIE, 'content-type': 'application/json' }, J({ identity: pick }));
169. await post('w-sel-exchange', '/api/exchange', { key: K, secret, code: val(r).code });
170. await save('w-sel-choose-out', 'POST', '/api/selector/choose?key=' + K, { origin: ORIGIN, 'content-type': 'application/json' }, J({ identity: pick }));
171. // notifications
172. await post('w-notify', '/api/notify', { key: K, secret, identity: SHORT, name: 'comment', text: 'Hello\nworld', icon: 'https://rd.example.org/i.png', link: 'https://rd.example.org/x' });
173. await post('w-notify-urgent', '/api/notify', { key: K, secret, identity: SHORT.toUpperCase(), name: 'mention', text: 'urgent one', urgent: true });
174. await post('w-notify-unregistered', '/api/notify', { key: K, secret, identity: SHORT, name: 'other', text: 'not registered' });
175. await post('w-notify-unknown', '/api/notify', { key: K, secret, identity: 'zzzzz', name: 'comment', text: 'x' });
176. await post('w-notify-badurl', '/api/notify', { key: K, secret, identity: SHORT, name: 'comment', text: 'x', icon: 'javascript:x' });
177. r = await save('w-page-inbox', 'GET', '/inbox', { cookie: COOKIE });
178. const conn = [...r.body.matchAll(/\/inbox\/(0m[0-9a-z]{10})/g)].map(m => m[1]).filter(c => !conns.includes(c))[0];
179. const note = [...r.body.matchAll(/value="(0m[0-9a-z]{10})"/g)].map(m => m[1])[0];
180. await face('inboxMark', 'inboxMark', [note, true]);
181. await face('inboxMark-bad', 'inboxMark', [note, 'yes']);
182. await save('w-page-conn', 'GET', '/inbox/' + conn, { cookie: COOKIE });
183. await face('inboxSwitch', 'inboxSwitch', [conn, 'comment', 'push', false]);
184. await face('inboxSwitch-unreg', 'inboxSwitch', [conn, 'other', 'push', true]);
185. await face('inboxSwitch-email', 'inboxSwitch', [conn, 'other', 'email', false]);
186. await face('inboxOverride', 'inboxOverride', [conn, 'active', true]);
187. await face('inboxOverride-push', 'inboxOverride', [conn, 'push', true]);
188. await post('w-notify-after', '/api/notify', { key: K, secret, identity: SHORT, name: 'comment', text: 'after the switches', urgent: true });
189. await post('w-kinds-again', '/api/kinds', { key: K, secret, kinds: [{ name: 'other', push: true, email: true }] });
190. await save('w-page-conn-after', 'GET', '/inbox/' + conn, { cookie: COOKIE });
191. await save('w-page-inbox-after', 'GET', '/inbox', { cookie: COOKIE });
192. // invites
193. r = await post('w-invite', '/api/invites', { key: K, secret, project: 'proj', role: 'member', return: ORIGIN + '/joined', uses: 2, days: 3 });
194. const inv = val(r);
195. await post('w-invite-mail', '/api/invites', { key: K, secret, project: 'proj', role: 'viewer', return: ORIGIN + '/joined', email: '[email protected]' });
196. await post('w-invite-bad', '/api/invites', { key: K, secret, project: 'proj', role: 'member', return: 'https://evil.example/' });
197. await post('w-invites-list', '/api/invites/list', { key: K, secret });
198. r = await save('w-invite-open', 'GET', '/invite/' + inv.url.split('/').pop());
199. rid = r.headers.location.split('/').pop();
200. await save('w-invite-signin', 'GET', '/signin/' + rid, { cookie: COOKIE });
201. await face('chooseIdentity-invite', 'chooseIdentity', [rid, IDENTITY]);
202. await post('w-invite-get', '/api/invites/get', { key: K, secret, id: inv.id });
203. await post('w-invite-revoke', '/api/invites/revoke', { key: K, secret, id: inv.id });
204. await post('w-invite-revoke-again', '/api/invites/revoke', { key: K, secret, id: inv.id });
205. await save('w-invite-open-revoked', 'GET', '/invite/' + inv.url.split('/').pop());
206. await post('w-migrate-ids', '/api/migrate-ids', { key: K, secret });
207. await post('w-migrate-ids-finish', '/api/migrate-ids', { key: K, secret, finish: true });
208. // the OTP login: a new address (creates the account), then the creator's own
209. for (const [tag, email] of [['new', '[email protected]'], ['creator', EMAIL]]) {
210. await post('w-code-' + tag, '/api/code', { email: email.toUpperCase() });
211. f = await face('rememberPending-' + tag, 'rememberPending', [email], null);
212. const ck = f.cookie;
213. await save('w-codepage-' + tag, 'GET', '/code', { cookie: ck });
214. await save('w-home-pending-' + tag, 'GET', '/', { cookie: ck });
215. await face('verifyCode-wrong-' + tag, 'verifyCode', [email, '000000', 'Pacific/Auckland'], ck);
216. await face('verifyCode-' + tag, 'verifyCode', [email, lastCode(email), 'Pacific/Auckland'], ck);
217. await save('w-home-signedin-' + tag, 'GET', '/', { cookie: ck });
218. await save('w-codepage-after-' + tag, 'GET', '/code', { cookie: ck });
219. await face('rememberPending-none-' + tag, 'rememberPending', [email], ck);
220. await face('signOut-' + tag, 'signOut', [], ck);
221. await save('w-home-signedout-' + tag, 'GET', '/', { cookie: ck });
222. }
223. // an app login with the code step on /signin/<rid>/code, and "other address"
224. r = await save('w-login2', 'GET', '/login?key=' + K + '&return=' + encodeURIComponent(ORIGIN + '/b'));
225. rid = r.headers.location.split('/').pop();
226. await post('w-code-app', '/api/code', { email: '[email protected]' });
227. f = await face('rememberPending-app', 'rememberPending', ['[email protected]'], null);
228. await save('w-signin-codepage', 'GET', '/signin/' + rid + '/code', { cookie: f.cookie });
229. await face('forgetPending', 'forgetPending', [], f.cookie);
230. await save('w-signin-codepage-forgot', 'GET', '/signin/' + rid + '/code', { cookie: f.cookie });
231. for (let i = 0; i < 3; i++) await post('w-code-limit-' + i, '/api/code', { email: '[email protected]' });
232. await post('w-code-ip', '/api/code', { email: '[email protected]' }, { 'x-client-ip': '2a01:4f9:3080:1126::1' });
233. // sign out everywhere (the new account, signed in again), then delete the app
234. await post('w-code-new2', '/api/code', { email: '[email protected]' });
235. f = await face('verifyCode-new2', 'verifyCode', ['[email protected]', lastCode('[email protected]'), 'UTC'], null);
236. await face('signOutAll', 'signOutAll', [], f.cookie);
237. await save('w-home-after-all', 'GET', '/', { cookie: f.cookie });
238. await face('inboxDisconnect', 'inboxDisconnect', [conn]);
239. await face('appDelete', 'appDelete', [app.id]);
240. await save('w-page-apps-after', 'GET', '/apps', { cookie: COOKIE });
241. await save('w-page-inbox-end', 'GET', '/inbox', { cookie: COOKIE });
242. await save('w-page-home-end', 'GET', '/', { cookie: COOKIE });
243. file('w-mail-sink', readFileSync(SINK, 'utf8').replace(/ \d{6}$/gm, ' CODE'));
244. file('w-mail-invites', existsSync(SINK + '.invites') ? readFileSync(SINK + '.invites', 'utf8') : '');
245. } finally {
246. srv.kill('SIGTERM'); await sleep(500); try { srv.kill('SIGKILL'); } catch {}
247. writeFileSync(join(OUT, '0000-server-log'), log);
248. }
249. // the stored state after the writes
250. const after = dumpOf(join(RUN, 'mpackdb'));
251. const sortRows = (rows) => rows.map(r => J(Object.fromEntries(Object.entries(r).sort()))).sort();
252. file('w-store', Object.keys(after).sort().map(t => '## ' + t + '\n' + sortRows(after[t]).join('\n')).join('\n'));
253. rmSync(RUN, { recursive: true, force: true });
254. console.log(`${n} files in ${OUT}`);