I am trying to migration the leveldb data into another database, now I am iterate the leveldb keys and put it into another database, I found the update key clock is not a number, this is the function of y-leveldb to genretate the key:
/**
* Create a unique key for a update message.
* We encode the result using `keyEncoding` which expects an array.
*
* @param {string} docName
* @param {number} clock must be unique
* @return {Array<string|number>}
*/
const createDocumentUpdateKey = (docName, clock) => ['v1', docName, 'update', clock];
when I read the key from database, some key look like this:
│ [2025-03-01T15:01:11.713] [ERROR] default - parse clock failedn, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,n │
│ [2025-03-01T15:01:14.512] [ERROR] default - parse clock failedo, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,o │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedp, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,p │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedq, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,q │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedr, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,r │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock faileds, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,s │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedt, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,t │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedu, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,u │
│ [2025-03-01T15:01:14.515] [ERROR] default - parse clock failedv, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,v │
│ [2025-03-01T15:01:14.516] [ERROR] default - parse clock failedw, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,w │
│ [2025-03-01T15:01:14.516] [ERROR] default - parse clock failedx, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,x │
│ [2025-03-01T15:01:15.512] [ERROR] default - parse clock failedy, part:v1, 80efe472ced04ce7823235a993bcdcc3,update,y
the key’s clock part is not a number, am I missing something? why did this happen?This is how I read the leveldb keys:
// @ts-ignore
import levelup from "levelup";
// @ts-ignore
import leveldown from "leveldown";
const leveldbPath = "/Users/xiaoqiangjiang/apps/texhub/yjs-storage-socketio";
import { PostgresqlPersistance } from "../../storage/adapter/postgresql/postgresql_persistance.js";
import logger from "../log4js_config.js";
const persistenceDir =
process.env.APP_ENV == "development" ? leveldbPath : process.env.YPERSISTENCE;
var db = levelup(leveldown(persistenceDir));
const postgresqlDb: PostgresqlPersistance = new PostgresqlPersistance();
export function iterateAllKeys() {
const keyStream = db.createKeyStream();
keyStream.on("data", (key: string) => {
db.get(key, async function (err: any, value: any) {
if (err) return logger.error("Ooops!", err);
const keyString = key.toString();
const controlChars = ["\u0002", "\u0001", "\u0006", "\u0000", "\u0005"];
const parts: string[] = splitByControlChars(keyString, controlChars);
if (parts.length > 2) {
handleGt2Keys(parts, value);
} else {
logger.info("less than 2", parts);
}
});
});
keyStream.on("end", () => {
logger.info("All keys have been iterated.");
});
keyStream.on("error", (err: Error) => {
logger.info("Error:", err);
});
}
async function handleGt2Keys(parts: string[], value: any) {
let keyMap: Map<string, string> = new Map<string, string>();
keyMap.set("version", parts[0]);
keyMap.set("docName", parts[1]);
keyMap.set("contentType", parts[2]);
if (parts[3]) {
let clock = parseInt(parts[3], 16);
if (isNaN(clock)) {
logger.error("parse clock failed" + parts[3] + ", part:" + parts);
}
keyMap.set("clock", isNaN(clock) ? "0" : clock.toString());
await postgresqlDb.storeUpdateWithSource(value, keyMap);
} else {
keyMap.set("clock", "0");
await postgresqlDb.storeUpdateWithSource(value, keyMap);
}
}
function splitByControlChars(str: string, controlChars: string[]): string[] {
const controlCharSet = new Set(controlChars);
const result: string[] = [];
let current = "";
for (let i = 0; i < str.length; i++) {
const char = str[i];
if (controlCharSet.has(char)) {
if (current) {
result.push(current);
current = "";
}
} else {
current += char;
}
}
if (current) {
result.push(current);
}
return result;
}