Walleye, managed.

import json, os, sys
import lancedb, pyarrow as pa

db = lancedb.connect(
    "db://" + os.environ["LAKEDAY_INSTANCE_ID"],  # the instance id is the database
    api_key=os.environ["LAKEDAY_TOKEN"],
    host_override="https://db.lakeday.ai",
)

schema = pa.schema([
    pa.field("id", pa.int64()),
    pa.field("body", pa.string()),
    pa.field("embedding", pa.list_(pa.float32(), 768)),
])
table = (
    db.open_table("events")
    if "events" in db.list_tables().tables
    else db.create_table("events", schema=schema)
)

rows = []
for line in sys.stdin:               # one JSON record per line
    event = json.loads(line)
    rows.append({
        "id": event["id"],
        "body": event["body"],
        "embedding": embed(event["body"]),
    })
    if len(rows) < 500:
        continue
    # Returns once the batch is durable in object storage.
    table.add(pa.Table.from_pylist(rows, schema=schema))
    rows = []

if rows:
    table.add(pa.Table.from_pylist(rows, schema=schema))

print(table.count_rows())
print(table.search(embed("checkout failed")).limit(5).to_list())

import * as lancedb from "@lancedb/lancedb";
import { createInterface } from "node:readline";

const db = await lancedb.connect(`db://${process.env.LAKEDAY_INSTANCE_ID}`, {
  apiKey: process.env.LAKEDAY_TOKEN,
  hostOverride: "https://db.lakeday.ai",
});

const names = await db.tableNames();
let table = names.includes("events") ? await db.openTable("events") : null;
let rows = [];

async function flush() {
  if (rows.length === 0) return;
  // Resolves once the batch is durable in object storage.
  if (table) await table.add(rows);
  else table = await db.createTable("events", rows);
  rows = [];
}

for await (const line of createInterface({ input: process.stdin })) {
  const event = JSON.parse(line);
  rows.push({
    id: event.id,
    body: event.body,
    embedding: await embed(event.body),
  });
  if (rows.length === 500) await flush();
}
await flush();

const query = table.search(await embed("checkout failed"));
console.log(await table.countRows());
console.log(await query.limit(5).toArray());