ident

8.5 KB

1. // tools/migrate-009.hl — ONE-OFF MIGRATION (ticket ident #10 / old #39, mission 009,
2. // 2026-09-24): the old `*id` tables (`storage/ident-<table>.*`) into the new ones
3. // (`storage/mpackdb/<table>.*`, pk `@id` UUID), plus the session files.
4. //
5. // * every row of every table (accounts, identities, otp, sends, apps, connections,
6. // requests, grants) is copied field by field; `id` becomes a new UUID and the old
7. // PUBLIC id (old mpackdb id + 1 — what the pages, faces and sessions used) is kept
8. // as `oldId`.
9. // * references are rewritten to the new UUIDs: identities.account, apps.owner,
10. // connections.app / .identity / .pair ('<app>:<identity>'), requests.app,
11. // grants.app / .identity.
12. // * WHAT APPS HOLD STAYS BYTE-IDENTICAL: apps.apiKey, apps.secretHash,
13. // connections.appIdentity, grants.appIdentity/hash, requests.rid are copied as is.
14. // * SESSIONS (optional, IDENT_SESSIONS_MIGRATE = the .sessions dir): a file whose
15. // `"user":{"id":<old account number>}` names a migrated account gets that account's
16. // UUID instead (only that substring changes). Anonymous sessions and ones already
17. // migrated are left alone.
18. //
19. // IDEMPOTENT: a row whose oldId is already in its new table is not written again, a
20. // session whose user id is already a string is skipped — a second run prints "0 new".
21. // The OLD FILES are opened with compact = false and never written by this tool — but
22. // hl:mpackdb rewrites a table's files on open (hybriel #40): RUN IT ON A COPY of them.
23. //
24. // STOP THE SERVER FIRST (it holds the tables and the sessions in memory), back up, then:
25. //
26. // IDENT_OLD_STORAGE=/abs/copy/ident IDENT_STORAGE=$PWD/storage/mpackdb \
27. // IDENT_SESSIONS_MIGRATE=$PWD/.sessions bin/hybriel tools/migrate-009.hl
28. //
29. // All paths ABSOLUTE (Hybriel resolves relative paths against tools/).
30. import { MPackDB } from 'hl:mpackdb'
31. import { env } from 'hl:proc'
32. import { exists, listDir, readFile, writeFile } from 'hl:fs'
33. import { dir, countOf } from '../lib/util.hl'
34. import { accountsTable } from '../lib/accounts.hl'
35. import { identitiesTable } from '../lib/identities.hl'
36. import { otpTable, sendsTable } from '../lib/login.hl'
37. import { appsTable, connectionsTable, requestsTable, grantsTable } from '../lib/apps.hl'
39. oldPrefix = env('IDENT_OLD_STORAGE')
40. if (oldPrefix == null || !oldPrefix.startsWith('/') || env('IDENT_STORAGE') == null || !env('IDENT_STORAGE').startsWith('/')) {
41. hlError('set IDENT_OLD_STORAGE (old table prefix, e.g. /abs/copy/ident) and IDENT_STORAGE (new table directory, e.g. $PWD/storage/mpackdb), both ABSOLUTE')
42. }
43. tableNames = ['accounts' 'identities' 'otp' 'sends' 'apps' 'connections' 'requests' 'grants']
44. for (t of tableNames) {
45. if (!exists(oldPrefix + '-' + t + '.mpack')) { hlError('no old table ' + oldPrefix + '-' + t + '.mpack') }
46. }
48. // the old tables, exactly as the old store.hl / apps.hl declared them; compact = false
49. oldAccounts = new MPackDB(file = oldPrefix + '-accounts.db', primaryKey = '*id', indexes = ['!email'], compact = false)
50. oldIdentities = new MPackDB(file = oldPrefix + '-identities.db', primaryKey = '*id', indexes = ['account'], compact = false)
51. oldOtp = new MPackDB(file = oldPrefix + '-otp.db', primaryKey = '*id', indexes = ['email'], compact = false)
52. oldSends = new MPackDB(file = oldPrefix + '-sends.db', primaryKey = '*id', indexes = ['email'], compact = false)
53. oldApps = new MPackDB(file = oldPrefix + '-apps.db', primaryKey = '*id', indexes = ['owner' '!apiKey'], compact = false)
54. oldConnections = new MPackDB(file = oldPrefix + '-connections.db', primaryKey = '*id', indexes = ['!pair' 'app' '!appIdentity'], compact = false)
55. oldRequests = new MPackDB(file = oldPrefix + '-requests.db', primaryKey = '*id', indexes = ['!rid'], compact = false)
56. oldGrants = new MPackDB(file = oldPrefix + '-grants.db', primaryKey = '*id', indexes = ['!hash'], compact = false)
58. static all = (table) => {
59. out = []
60. found = table.find(null, null)
61. if (countOf(found) > 0) { for (r of found) { out.push(r) } }
62. return out
63. }
64. // old public id (as a string key) → new UUID, from the rows already migrated
65. static mapOf = (table) => {
66. m = {}
67. for (r of all(table)) { if (r.oldId != null) { m['' + r.oldId] = r.id } }
68. return m
69. }
70. // every field of `rec` but `id`, then `extra` on top
71. static copied = (rec, extra) => {
72. out = {}
73. for (k of rec.keys()) { if (k != 'id') { out[k] = rec[k] } }
74. for (k of extra.keys()) { out[k] = extra[k] }
75. return out
76. }
78. failed = 0
79. report = []
80. // copy one old table: `refs(rec)` answers the extra fields (the rewritten references),
81. // or { skip = reason } when a reference cannot be resolved
82. migrateTable = (name, oldTable, newTable, refs) => {
83. have = mapOf(newTable)
84. let fresh = 0
85. let known = 0
86. for (r of all(oldTable)) {
87. let oldId = r.id + 1
88. if (have['' + oldId] != null) {
89. known = known + 1
90. } else {
91. let extra = refs(r)
92. if (extra.skip != null) {
93. console.log('SKIP ' + name + ' old ' + oldId + ': ' + extra.skip)
94. failed = failed + 1
95. } else {
96. extra.oldId = oldId
97. let id = newTable.put(copied(r, extra))
98. if (id == null) {
99. console.log('FAIL ' + name + ' old ' + oldId + ': ' + newTable.lastError())
100. failed = failed + 1
101. } else {
102. have['' + oldId] = id
103. fresh = fresh + 1
104. }
105. }
106. }
107. }
108. report.push(name + ' ' + fresh + ' new, ' + known + ' already there (old ' + oldTable.count() + ' → new ' + newTable.count() + ')')
109. return have
110. }
112. none = (r) => { return {} }
113. acc = migrateTable('accounts', oldAccounts, accountsTable, none)
114. ids = migrateTable('identities', oldIdentities, identitiesTable, (r) => {
115. a = acc['' + r.account]
116. return a == null ? { skip = 'its account ' + r.account + ' is gone' } : { account = a }
117. })
118. migrateTable('otp', oldOtp, otpTable, none)
119. migrateTable('sends', oldSends, sendsTable, none)
120. apps = migrateTable('apps', oldApps, appsTable, (r) => {
121. o = acc['' + r.owner]
122. return o == null ? { skip = 'its owner ' + r.owner + ' is gone' } : { owner = o }
123. })
124. // a connection of a DELETED identity stays (as it did before: its id is still the app's),
125. // its identity reads 'gone:<old id>' so the pair stays unique and matches no identity
126. migrateTable('connections', oldConnections, connectionsTable, (r) => {
127. a = apps['' + r.app]
128. if (a == null) { return { skip = 'its app ' + r.app + ' is gone' } }
129. let i = ids['' + r.identity]
130. if (i == null) { i = 'gone:' + r.identity }
131. return { app = a identity = i pair = a + ':' + i }
132. })
133. migrateTable('requests', oldRequests, requestsTable, (r) => {
134. a = apps['' + r.app]
135. return a == null ? { skip = 'its app ' + r.app + ' is gone' } : { app = a }
136. })
137. migrateTable('grants', oldGrants, grantsTable, (r) => {
138. a = apps['' + r.app]
139. i = ids['' + r.identity]
140. if (a == null) { return { skip = 'its app ' + r.app + ' is gone' } }
141. return { app = a identity = i == null ? 'gone:' + r.identity : i }
142. })
144. // ---- the session files --------------------------------------------------------------
145. sessDir = env('IDENT_SESSIONS_MIGRATE')
146. static digitsAt = (s, p0) => {
147. let i = p0
148. while (i < s.length && s.charCodeAt(i) >= 48 && s.charCodeAt(i) <= 57) { i = i + 1 }
149. return i
150. }
151. if (sessDir != null && sessDir != '') {
152. if (!sessDir.startsWith('/') || !exists(sessDir)) { hlError('IDENT_SESSIONS_MIGRATE must be an existing ABSOLUTE directory') }
153. let moved = 0
154. let already = 0
155. let anonymous = 0
156. let unknown = 0
157. pre = '"user":{"id":'
158. for (e of listDir(sessDir)) {
159. let raw = readFile(e.path)
160. let at = raw == null ? -1 : raw.indexOf(pre)
161. if (at < 0) {
162. anonymous = anonymous + 1
163. } else {
164. let p0 = at + pre.length
165. let end = digitsAt(raw, p0)
166. if (end == p0) {
167. already = already + 1 // a string id: migrated before
168. } else if (raw[end] != '}') {
169. console.log('SESSION ' + e.name + ': unexpected user shape, left alone')
170. unknown = unknown + 1
171. } else {
172. let n = raw.slice(p0, end)
173. let to = acc[n]
174. if (to == null) {
175. console.log('SESSION ' + e.name + ': account ' + n + ' is not migrated, left alone (it reads as signed out)')
176. unknown = unknown + 1
177. } else {
178. let next = raw.slice(0, p0) + '"' + to + '"' + raw.slice(end, raw.length)
179. let check = JSON.parse(next)
180. if (check == null || check.user == null || check.user.id != to) {
181. console.log('SESSION ' + e.name + ': rewrite did not read back, left alone')
182. failed = failed + 1
183. } else {
184. writeFile(e.path, next, 384)
185. moved = moved + 1
186. }
187. }
188. }
189. }
190. }
191. report.push('sessions ' + moved + ' new, ' + already + ' already there, ' + anonymous + ' anonymous, ' + unknown + ' left alone (' + sessDir + ')')
192. }
194. for (line of report) { console.log('migrate: ' + line) }
195. console.log('migrate: ' + failed + ' failed (' + oldPrefix + ' → ' + dir + ')')