ident

8.4 KB

1. // tools/migrate-009.hl — ONE-OFF MIGRATION (ticket ident #10 / old #39, mission 009,
2. // 2026-09-24): the old `*id` tables (`storage/ident-<table>.*`) into the new ones
3. // (`storage/mpackdb/<table>.*`, pk `@id` UUID), plus the session files.
4. //
5. // * every row of every table (accounts, identities, otp, sends, apps, connections,
6. // requests, grants) is copied field by field; `id` becomes a new UUID and the old
7. // PUBLIC id (old mpackdb id + 1 — what the pages, faces and sessions used) is kept
8. // as `oldId`.
9. // * references are rewritten to the new UUIDs: identities.account, apps.owner,
10. // connections.app / .identity / .pair ('<app>:<identity>'), requests.app,
11. // grants.app / .identity.
12. // * WHAT APPS HOLD STAYS BYTE-IDENTICAL: apps.apiKey, apps.secretHash,
13. // connections.appIdentity, grants.appIdentity/hash, requests.rid are copied as is.
14. // * SESSIONS (optional, IDENT_SESSIONS_MIGRATE = the .sessions dir): a file whose
15. // `"user":{"id":<old account number>}` names a migrated account gets that account's
16. // UUID instead (only that substring changes). Anonymous sessions and ones already
17. // migrated are left alone.
18. //
19. // IDEMPOTENT: a row whose oldId is already in its new table is not written again, a
20. // session whose user id is already a string is skipped — a second run prints "0 new".
21. // The OLD FILES are opened with compact = false and never written by this tool — but
22. // hl:mpackdb rewrites a table's files on open (hybriel #40): RUN IT ON A COPY of them.
23. //
24. // STOP THE SERVER FIRST (it holds the tables and the sessions in memory), back up, then:
25. //
26. // IDENT_OLD_STORAGE=/abs/copy/ident IDENT_STORAGE=$PWD/storage/mpackdb \
27. // IDENT_SESSIONS_MIGRATE=$PWD/.sessions bin/hybriel tools/migrate-009.hl
28. //
29. // All paths ABSOLUTE (Hybriel resolves relative paths against tools/).
30. import { MPackDB } from 'hl:mpackdb'
31. import { env } from 'hl:proc'
32. import { exists, listDir, readFile, writeFile } from 'hl:fs'
33. import { dir, accountsTable, identitiesTable, otpTable, sendsTable, countOf } from '../store.hl'
34. import { appsTable, connectionsTable, requestsTable, grantsTable } from '../apps.hl'
36. oldPrefix = env('IDENT_OLD_STORAGE')
37. if (oldPrefix == null || !oldPrefix.startsWith('/') || env('IDENT_STORAGE') == null || !env('IDENT_STORAGE').startsWith('/')) {
38. hlError('set IDENT_OLD_STORAGE (old table prefix, e.g. /abs/copy/ident) and IDENT_STORAGE (new table directory, e.g. $PWD/storage/mpackdb), both ABSOLUTE')
39. }
40. tableNames = ['accounts' 'identities' 'otp' 'sends' 'apps' 'connections' 'requests' 'grants']
41. for (t of tableNames) {
42. if (!exists(oldPrefix + '-' + t + '.mpack')) { hlError('no old table ' + oldPrefix + '-' + t + '.mpack') }
43. }
45. // the old tables, exactly as the old store.hl / apps.hl declared them; compact = false
46. oldAccounts = new MPackDB(file = oldPrefix + '-accounts.db', primaryKey = '*id', indexes = ['!email'], compact = false)
47. oldIdentities = new MPackDB(file = oldPrefix + '-identities.db', primaryKey = '*id', indexes = ['account'], compact = false)
48. oldOtp = new MPackDB(file = oldPrefix + '-otp.db', primaryKey = '*id', indexes = ['email'], compact = false)
49. oldSends = new MPackDB(file = oldPrefix + '-sends.db', primaryKey = '*id', indexes = ['email'], compact = false)
50. oldApps = new MPackDB(file = oldPrefix + '-apps.db', primaryKey = '*id', indexes = ['owner' '!apiKey'], compact = false)
51. oldConnections = new MPackDB(file = oldPrefix + '-connections.db', primaryKey = '*id', indexes = ['!pair' 'app' '!appIdentity'], compact = false)
52. oldRequests = new MPackDB(file = oldPrefix + '-requests.db', primaryKey = '*id', indexes = ['!rid'], compact = false)
53. oldGrants = new MPackDB(file = oldPrefix + '-grants.db', primaryKey = '*id', indexes = ['!hash'], compact = false)
55. static all = (table) => {
56. let out = []
57. let found = table.find(null, null)
58. if (countOf(found) > 0) { for (r of found) { out.push(r) } }
59. return out
60. }
61. // old public id (as a string key) → new UUID, from the rows already migrated
62. static mapOf = (table) => {
63. let m = {}
64. for (r of all(table)) { if (r.oldId != null) { m['' + r.oldId] = r.id } }
65. return m
66. }
67. // every field of `rec` but `id`, then `extra` on top
68. static copied = (rec, extra) => {
69. let out = {}
70. for (k of rec.keys()) { if (k != 'id') { out[k] = rec[k] } }
71. for (k of extra.keys()) { out[k] = extra[k] }
72. return out
73. }
75. failed = 0
76. report = []
77. // copy one old table: `refs(rec)` answers the extra fields (the rewritten references),
78. // or { skip = reason } when a reference cannot be resolved
79. migrateTable = (name, oldTable, newTable, refs) => {
80. let have = mapOf(newTable)
81. let fresh = 0
82. let known = 0
83. for (r of all(oldTable)) {
84. let oldId = r.id + 1
85. if (have['' + oldId] != null) {
86. known = known + 1
87. } else {
88. let extra = refs(r)
89. if (extra.skip != null) {
90. console.log('SKIP ' + name + ' old ' + oldId + ': ' + extra.skip)
91. failed = failed + 1
92. } else {
93. extra.oldId = oldId
94. let id = newTable.put(copied(r, extra))
95. if (id == null) {
96. console.log('FAIL ' + name + ' old ' + oldId + ': ' + newTable.lastError())
97. failed = failed + 1
98. } else {
99. have['' + oldId] = id
100. fresh = fresh + 1
101. }
102. }
103. }
104. }
105. report.push(name + ' ' + fresh + ' new, ' + known + ' already there (old ' + oldTable.count() + ' → new ' + newTable.count() + ')')
106. return have
107. }
109. none = (r) => { return {} }
110. acc = migrateTable('accounts', oldAccounts, accountsTable, none)
111. ids = migrateTable('identities', oldIdentities, identitiesTable, (r) => {
112. let a = acc['' + r.account]
113. return a == null ? { skip = 'its account ' + r.account + ' is gone' } : { account = a }
114. })
115. migrateTable('otp', oldOtp, otpTable, none)
116. migrateTable('sends', oldSends, sendsTable, none)
117. apps = migrateTable('apps', oldApps, appsTable, (r) => {
118. let o = acc['' + r.owner]
119. return o == null ? { skip = 'its owner ' + r.owner + ' is gone' } : { owner = o }
120. })
121. // a connection of a DELETED identity stays (as it did before: its id is still the app's),
122. // its identity reads 'gone:<old id>' so the pair stays unique and matches no identity
123. migrateTable('connections', oldConnections, connectionsTable, (r) => {
124. let a = apps['' + r.app]
125. if (a == null) { return { skip = 'its app ' + r.app + ' is gone' } }
126. let i = ids['' + r.identity]
127. if (i == null) { i = 'gone:' + r.identity }
128. return { app = a identity = i pair = a + ':' + i }
129. })
130. migrateTable('requests', oldRequests, requestsTable, (r) => {
131. let a = apps['' + r.app]
132. return a == null ? { skip = 'its app ' + r.app + ' is gone' } : { app = a }
133. })
134. migrateTable('grants', oldGrants, grantsTable, (r) => {
135. let a = apps['' + r.app]
136. let i = ids['' + r.identity]
137. if (a == null) { return { skip = 'its app ' + r.app + ' is gone' } }
138. return { app = a identity = i == null ? 'gone:' + r.identity : i }
139. })
141. // ---- the session files --------------------------------------------------------------
142. sessDir = env('IDENT_SESSIONS_MIGRATE')
143. static digitsAt = (s, p0) => {
144. let i = p0
145. while (i < s.length && s.charCodeAt(i) >= 48 && s.charCodeAt(i) <= 57) { i = i + 1 }
146. return i
147. }
148. if (sessDir != null && sessDir != '') {
149. if (!sessDir.startsWith('/') || !exists(sessDir)) { hlError('IDENT_SESSIONS_MIGRATE must be an existing ABSOLUTE directory') }
150. let moved = 0
151. let already = 0
152. let anonymous = 0
153. let unknown = 0
154. let pre = '"user":{"id":'
155. for (e of listDir(sessDir)) {
156. let raw = readFile(e.path)
157. let at = raw == null ? -1 : raw.indexOf(pre)
158. if (at < 0) {
159. anonymous = anonymous + 1
160. } else {
161. let p0 = at + pre.length
162. let end = digitsAt(raw, p0)
163. if (end == p0) {
164. already = already + 1 // a string id: migrated before
165. } else if (raw[end] != '}') {
166. console.log('SESSION ' + e.name + ': unexpected user shape, left alone')
167. unknown = unknown + 1
168. } else {
169. let n = raw.slice(p0, end)
170. let to = acc[n]
171. if (to == null) {
172. console.log('SESSION ' + e.name + ': account ' + n + ' is not migrated, left alone (it reads as signed out)')
173. unknown = unknown + 1
174. } else {
175. let next = raw.slice(0, p0) + '"' + to + '"' + raw.slice(end, raw.length)
176. let check = JSON.parse(next)
177. if (check == null || check.user == null || check.user.id != to) {
178. console.log('SESSION ' + e.name + ': rewrite did not read back, left alone')
179. failed = failed + 1
180. } else {
181. writeFile(e.path, next, 384)
182. moved = moved + 1
183. }
184. }
185. }
186. }
187. }
188. report.push('sessions ' + moved + ' new, ' + already + ' already there, ' + anonymous + ' anonymous, ' + unknown + ' left alone (' + sessDir + ')')
189. }
191. for (line of report) { console.log('migrate: ' + line) }
192. console.log('migrate: ' + failed + ' failed (' + oldPrefix + ' → ' + dir + ')')