New update turn to GC

jiangxiaoqiang · March 20, 2025, 11:16am

I have a document sync by yjs. Today when I update the yjs document, I found the new update content did not append to the ydoc. Then I tracing into the source code and found the all new yjs update auto turned to GC, this is the clock info that I stored in PostgreSQL 15:

INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",94]', decode('0101FCBCA2920601000100','hex'), '����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 94, 'ws', '2025-03-20 10:48:06.916');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",93]', decode('0101FCBCA2920600000100','hex'), '����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 93, 'ws', '2025-03-20 10:46:20.567');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",92]', decode('0101F8B3FFDC0300000100','hex'), '����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 92, 'ws', '2025-03-20 10:45:09.176');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",91]', decode('0101D3CEC68E0902000100','hex'), '��Ǝ	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 91, 'ws', '2025-03-20 10:44:12.660');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",90]', decode('0101D3CEC68E0901000100','hex'), '��Ǝ	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 90, 'ws', '2025-03-20 10:44:08.380');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",89]', decode('0101D3CEC68E0900000100','hex'), '��Ǝ	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 89, 'ws', '2025-03-20 10:43:18.847');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",88]', decode('0101A0A9A5DE0D00000100','hex'), '����
', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 88, 'ws', '2025-03-20 10:42:57.901');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",87]', decode('0101CBBEE7DF0301000100','hex'), '˾��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 87, 'ws', '2025-03-20 10:42:54.560');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",86]', decode('0101CBBEE7DF0300000100','hex'), '˾��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 86, 'ws', '2025-03-20 10:42:51.676');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",85]', decode('0101C1D283FF0E00000100','hex'), '�҃�', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 85, 'ws', '2025-03-20 10:42:48.830');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",84]', decode('0101FEB3C4F00900000100','hex'), '����	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 84, 'ws', '2025-03-20 10:42:46.057');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",83]', decode('0101AEB2CACF0201000100','hex'), '����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 83, 'ws', '2025-03-20 10:42:43.567');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",82]', decode('0101AEB2CACF0200000100','hex'), '����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 82, 'ws', '2025-03-20 10:42:39.512');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",81]', decode('0101EFEC99870500000100','hex'), '�왇', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 81, 'ws', '2025-03-20 10:42:34.993');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",80]', decode('0101F7BBE0A70400000100','hex'), '���', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 80, 'ws', '2025-03-20 10:42:32.107');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",79]', decode('0101D7A9B8EC0509000100','hex'), 'ש��	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 79, 'ws', '2025-03-20 10:42:30.125');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",78]', decode('0101D7A9B8EC0508000100','hex'), 'ש��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 78, 'ws', '2025-03-20 10:42:30.047');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",77]', decode('0101D7A9B8EC0507000100','hex'), 'ש��a', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 77, 'ws', '2025-03-20 10:42:29.971');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",76]', decode('0101D7A9B8EC0506000100','hex'), 'ש��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 76, 'ws', '2025-03-20 10:42:29.911');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",75]', decode('0101D7A9B8EC0505000100','hex'), 'ש��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 75, 'ws', '2025-03-20 10:42:29.842');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",74]', decode('0101D7A9B8EC0504000100','hex'), 'ש��', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 74, 'ws', '2025-03-20 10:42:29.751');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",73]', decode('0104D7A9B8EC050027010575736572730E6A69616E677869616F7169616E67012700D7A9B8EC050003696473002700D7A9B8EC0500026473000800D7A9B8EC0501017D97D3F0D80B00','hex'), 'ש��''usersjiangxiaoqiang''ש��ids''ש��dsש��}����', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 73, 'ws', '2025-03-20 10:42:29.394');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",72]', decode('0000','hex'), '', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 72, 'ws', '2025-03-19 04:02:06.280');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",71]', decode('0000','hex'), '', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 71, 'ws', '2025-03-17 14:40:47.557');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",70]', decode('0000','hex'), '', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 70, 'ws', '2025-03-17 14:02:38.788');
INSERT INTO public.tex_sync
("key", value, plain_value, "version", content_type, doc_name, clock, "source", created_time)
VALUES('["v1","745c2986288a4478bf44e14746d22d68","update",69]', decode('1001FEE291F30E0088E7FEB8CF0700017B41EDCC8E2FC0000001F782B8CD0B00000102D6E7D1990B0000018184E6D0B001090C01F8B98FA50A0088CDAEF0D20703017B41E4947B9F0000000792A3B9E60900000E84D6E7D1990B0C0B31313131313131313131318492A3B9E60918120A6671716765677732323134313234323132C492A3B9E6091892A3B9E6091904323232328492A3B9E6092A023232C492A3B9E6092E92A3B9E60919213132313231313231323132313231323132313231323131323132313231323132318492A3B9E609300832323271313131310181BCA2D2080000010197D1AC950800000104CDAEF0D2070027010575736572730E6A69616E677869616F7169616E67012700CDAEF0D2070003696473002700CDAEF0D20700026473000800CDAEF0D20701017D8DDDE0A50F01E7FEB8CF070088F8B98FA50A00017DA7FDF19E0F03F8A5B1B0060021010575736572730E6A69616E677869616F7169616E670100080300CDAEF0D20702010004E3F8E485060021010575736572730E6A69616E677869616F7169616E6701000381D4B084C702080581D4B084C702090102BFA1D98C040021010575736572730E6A69616E677869616F7169616E6701000302A199BDCC020021010575736572730E6A69616E677869616F7169616E6701000302C28EC6C7020021010575736572730E6A69616E677869616F7169616E6701000704D4B084C7020021010575736572730E6A69616E677869616F7169616E6701000381FEE291F30E000581F8A5B1B00609010484E6D0B0010021010575736572730E6A69616E677869616F7169616E6701000604012037343563323938363238386134343738626634346531343734366432326436380231328184E6D0B00108010CF782B8CD0B010001D6E7D1990B01000D92A3B9E60901000E81BCA2D20801000197D1AC9508010001F8A5B1B006010009E3F8E4850601000ABFA1D98C04010004A199BDCC02010004C28EC6C702010008D4B084C70201000A84E6D0B0010200070901','hex'), '��������a{A�̎/�������љ���а	����
�ͮ��a{A�{�a����	���љ11111111111�����	
fqqgegw2214124212Ē���	����	2222�����	*22Ē���	.����	!121211212121212121212112121212121�����	0222q1111�����Ѭ�ͮ��a''usersjiangxiaoqiang''ͮ��aids''ͮ��adsͮ��a}�������a�����
}�������!usersjiangxiaoqiangͮ��a���!usersjiangxiaoqiang�԰���԰��	��ٌ!usersjiangxiaoqiang����!usersjiangxiaoqiang��!usersjiangxiaoqianga԰��!usersjiangxiaoqiang���������	��а!usersjiangxiaoqiang 745c2986288a4478bf44e14746d22d6812���а������љ
����	�����Ѭ�����	���
��ٌ������԰��
��аa	', 'v1', 'update', '745c2986288a4478bf44e14746d22d68', 69, 'ws', '2025-03-17 11:28:54.292');

and this is the PostgreSQL 15 DDL:

CREATE TABLE public.tex_sync (

id int8 GENERATED ALWAYS AS IDENTITY( INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1 NO CYCLE) NOT NULL,

"key" varchar NOT NULL,

value bytea NULL,

plain_value varchar NULL,

"version" varchar NULL,

content_type varchar NULL,

doc_name varchar NULL,

clock int4 NULL,

"source" varchar NULL,

created_time timestamp DEFAULT CURRENT_TIMESTAMP NULL,

CONSTRAINT tex_sync_unique UNIQUE (key)

);

and this is the select command:

// @ts-ignore
import * as Y from "yjs";
import { dbConfig } from "./db_config.js";
import { TeXSync } from "./tex_sync.js";
import * as binary from "lib0/binary.js";
import pg, { QueryResult } from "pg";
const { Pool } = pg;

export class PostgresqlPersistance {
  pool: pg.Pool;

  constructor() {
    const pool = new Pool({
      host: "reddwarf-postgresql.reddwarf-storage.svc.cluster.local",
      port: "5432",
      database: "yjs",
      user: "postgres",
      password: "123456",
      max: 100,
      idleTimeoutMillis: 30000,
    });
    this.pool = pool;
  }

  async getPgBulkData(db: pg.Pool, opts: any, docName: string) {
    try {
      let col = [];
      col.push("id");
      col.push("clock");
      if (opts.values) {
        col.push("value");
      }
      if (opts.keys) {
        col.push("key");
      }
      let col_concat = col.join(",");
      const queryPart = "select " + col_concat;
      const fromPart = " from tex_sync ";
      const filterPart =
        " where doc_name = '" +
        docName +
        "' and content_type='update' and clock>=0 and clock < 4429";
      let orderPart = " order by clock asc";
      if (opts.reverse) {
        orderPart = " order by clock desc";
      }
      let limitPart = "";
      if (opts.limit) {
        limitPart = " limit " + opts.limit;
      }
      const sql = queryPart + fromPart + filterPart + orderPart + limitPart;
      let result: QueryResult<TeXSync> = await db.query(sql);
      return result.rows;
    } catch (err) {
      console.error("Query error:", err);
      throw err;
    }
  }

  async getYDoc(docName: string): Promise<Y.Doc> {
    const updates: Array<TeXSync> = await this.getPgBulkData(
      this.pool,
      { values: true, keys: false, reverse: false },
      docName
    );
    const ydoc = new Y.Doc();
    ydoc.transact(() => {
      try {
        for (let i = 0; i < updates.length; i++) {
          let update: TeXSync = updates[i];
          let updateVal: Uint8Array = update.value;
          Y.applyUpdate(ydoc, updateVal);
          let txt = ydoc.getText(docName);
          let txt1 = txt.toString();
          console.log("clock:" + update.clock + "," + txt1);
        }
      } catch (err) {
        console.error("apply update failed", err);
      }
    });
    let txt = ydoc.getText(docName);
    let txt1 = txt.toString();
    console.log(txt1);
    return ydoc;
  }
}

get the doc:

async getYDoc(docName: string): Promise<Y.Doc> {
    const updates: Array<TeXSync> = await this.getPgBulkData(
      this.pool,
      { values: true, keys: false, reverse: false },
      docName
    );
    const ydoc = new Y.Doc();
    ydoc.transact(() => {
      try {
        for (let i = 0; i < updates.length; i++) {
          let update: TeXSync = updates[i];
          let updateVal: Uint8Array = update.value;
          Y.applyUpdate(ydoc, updateVal);
          let txt = ydoc.getText(docName);
          let txt1 = txt.toString();
          console.log("clock:" + update.clock + "," + txt1);
        }
      } catch (err) {
        console.error("apply update failed", err);
      }
    });
    let txt = ydoc.getText(docName);
    let txt1 = txt.toString();
    console.log(txt1);
    return ydoc;
  }
}

when the clock > 73, today(2025-03-20) added data .all new apply items is GC object.

dmonad · April 21, 2025, 9:01pm

@jiangxiaoqiang Your posts seem a little off. Are they AI generated?

jiangxiaoqiang · April 22, 2025, 1:16pm

No, not AI generate. I have solved this issue, it seems like I mixed using the import ESM/CJS, now I turned the import using ESM.