1// Everything the site stores, readable by anyone (website/src/jev/DataTab.jsx,
2// the jev panel's data tab). jevstrudel is a playground shared in the Jev
3// community: what it keeps is public, people included, except secrets.
4//
5//   GET /jev/data                      totals for every store, the lobby and
6//                                      the live parties
7//   GET /jev/data/accounts             every account, newest first
8//   GET /jev/data/accounts/<id>        one: its passkeys and sessions by date,
9//                                      radio history, its takes (performances
10//                                      it played, newest first, at most
11//                                      ACCOUNT_TAKES), everything it wrote (held
12//                                      and pending too), today's Jev budget, its
13//                                      open tabs and connected apps (counted;
14//                                      named only to the account itself)
15//   GET /jev/data/votes                "do you agree with Jev?" counts
16//   GET /jev/data/reactions?song=      🔥/😴 counts
17//   GET /jev/data/performances?song=   every stored performance, with who played it
18//   GET /jev/data/performances/<id>    one, as stored: Jev's decision history
19//   GET /jev/data/songs                every listener song, public or not
20//   GET /jev/data/songs/<id>           one, every revision whole, with verdicts
21//   GET /jev/data/covers               every cover's record
22//   GET /jev/data/comments             every comment, with its verdict
23//   GET /jev/data/pitches              every pitch, with its verdict, who voted
24//                                      for it and when, and the listener songs
25//                                      answering it
26//   GET /jev/data/jobs                 the work Jev owes, and whose budget pays
27//   GET /jev/data/activity?before=     what is happening, newest first
28//                                      (activity.ts), cached ACTIVITY_CACHE_S
29//
30// An account's view also carries `publicSongs`, its songs as the song
31// browser lists them, for its profile (website/src/jev/Profile.jsx).
32//
33// Lists take `offset` and `limit` (at most MAX_LIMIT) and answer `total`.
34// Never a secret, anywhere: no session hash, passkey id or public key,
35// sign-in challenge, OAuth token, grant's props, or party room name or host
36// key hash; those are counted and dated. Held content is text, which the page
37// shows as text: its code is never played and a cover is served as an image
38// only by /jev/listeners/covers, which serves only public ones (content.ts).
39// Per visitor DATA_LIMIT; nothing is cached by the browser (no-store), since
40// it changes as people use the site. The KV and R2 counts cost list
41// operations, which the free plan allows 1000 a day (README.md, The hosted
42// MCP), so each isolate keeps them STORE_COUNTS_MS.
43import { d1Accounts } from './accounts-store';
44import { activity, ACTIVITY_DEFAULT, ACTIVITY_MAX } from './activity';
45import { cacheEntries } from './answer-cache';
46import { signedIn } from './auth';
47import { d1Content } from './content-store';
48import type { Env } from './env';
49import { d1Listening, d1ListeningReads } from './listening-store';
50import { appsOf, grantCounts, servers } from './oauth';
51import { directory } from './party';
52import { d1Radio, KEPT } from './radio';
53import { d1Votes } from './votes-store';
54import config from '../wrangler.json';
55
56export const DATA_PREFIX = '/jev/data';
57export const DEFAULT_LIMIT = 50;
58export const MAX_LIMIT = 100;
59export const MAX_OFFSET = 100_000;
60export const STORE_COUNTS_MS = 10 * 60_000;
61// An account's takes on its page: its recently played and its profile.
62export const ACCOUNT_TAKES = 50;
63// The activity feed changes as people write; a few seconds of cache
64// spares the database a tab of people all opening it at once.
65export const ACTIVITY_CACHE_S = 10;
66export const DATA_LIMIT = config.ratelimits.find((r) => r.name === 'DATA_LIMIT')!.simple;
67
68const ID = /^[A-Za-z0-9_-]{22}$/;
69const SITE_SONG = /^[a-z0-9-]{1,64}\/[a-z0-9-]{1,64}$/;
70
71const json = (status: number, body: unknown, headers: Record<string, string> = {}) =>
72  Response.json(body, { status, headers: { 'Cache-Control': 'no-store', ...headers } });
73const fail = (status: number, error: string, headers: Record<string, string> = {}) => json(status, { error }, headers);
74
75export function page(url: URL): { limit: number; offset: number } {
76  const n = (name: string, fallback: number, max: number) => {
77    const v = Number(url.searchParams.get(name) ?? fallback);
78    return Number.isInteger(v) && v >= 0 ? Math.min(v, max) : fallback;
79  };
80  return { limit: Math.max(1, n('limit', DEFAULT_LIMIT, MAX_LIMIT)), offset: n('offset', 0, MAX_OFFSET) };
81}
82
83// The counts that cost KV and R2 list operations, kept per isolate.
84type StoreCounts = {
85  answerCache: { entries: number; complete: boolean } | null;
86  oauthGrants: { grants: number; accounts: number; complete: boolean; byUser: Map<string, number> } | null;
87  coverObjects: { objects: number; bytes: number; complete: boolean } | null;
88  at: number;
89};
90let kept: StoreCounts | null = null;
91export const forgetStoreCounts = () => void (kept = null);
92
93async function storeCounts(env: Env): Promise<StoreCounts> {
94  const now = Date.now();
95  if (kept && now - kept.at < STORE_COUNTS_MS) return kept;
96  const failed = (what: string) => (e: unknown) => {
97    console.error({ event: 'jev.data', what, error: e instanceof Error ? e.message : String(e) });
98    return null;
99  };
100  const [answerCache, grants, coverObjects] = await Promise.all([
101    cacheEntries(env.CACHE).catch(failed('cache')),
102    grantCounts(env.OAUTH_KV).catch(failed('grants')),
103    r2Count(env.COVERS).catch(failed('covers')),
104  ]);
105  kept = {
106    answerCache,
107    oauthGrants: grants && { grants: grants.grants, accounts: grants.byUser.size, complete: grants.complete, byUser: grants.byUser },
108    coverObjects,
109    at: now,
110  };
111  return kept;
112}
113
114// The cover images in R2: at most 5 lists of 1000.
115async function r2Count(bucket: R2Bucket, pages = 5) {
116  let objects = 0;
117  let bytes = 0;
118  let cursor: string | undefined;
119  for (let i = 0; i < pages; i++) {
120    const list = await bucket.list({ prefix: 'covers/', cursor });
121    for (const o of list.objects) {
122      objects += 1;
123      bytes += o.size;
124    }
125    if (!list.truncated) return { objects, bytes, complete: true };
126    cursor = list.cursor;
127  }
128  return { objects, bytes, complete: false };
129}
130
131// A live object's answer, or null (and logged) when it fails: one store
132// down leaves the rest of the page.
133async function orNull<T>(what: string, p: Promise<T>): Promise<T | null> {
134  try {
135    return await p;
136  } catch (e) {
137    console.error({ event: 'jev.data', what, error: e instanceof Error ? e.message : String(e) });
138    return null;
139  }
140}
141
142export async function data(request: Request, env: Env, now: () => number = Date.now): Promise<Response> {
143  if (request.method !== 'GET' && request.method !== 'HEAD') return fail(405, 'GET only', { Allow: 'GET, HEAD' });
144  const visitor = request.headers.get('CF-Connecting-IP') ?? 'local';
145  if (!(await env.DATA_LIMIT.limit({ key: visitor })).success) {
146    return fail(429, 'too many reads; try again in a minute', { 'Retry-After': String(DATA_LIMIT.period) });
147  }
148  const url = new URL(request.url);
149  const [what, id, ...rest] = url.pathname.slice(DATA_PREFIX.length).split('/').filter(Boolean);
150  if (rest.length) return fail(404, 'no such data');
151  const { limit, offset } = page(url);
152  const accounts = d1Accounts(env.DB, now);
153  const content = d1Content(env.DB, now);
154  const listening = d1ListeningReads(env.DB);
155  const song = url.searchParams.get('song');
156  if (song !== null && !SITE_SONG.test(song)) return fail(400, 'song is a site song id, <theme>/<song>');
157
158  switch (what ?? '') {
159    case '': {
160      // the live objects and the lists meanwhile; the database's batches in turn
161      const others = Promise.all([
162        orNull('lobby', env.LOBBY.get(env.LOBBY.idFromName('lobby')).summary()),
163        orNull('parties', directory(env).live()),
164        storeCounts(env),
165      ]);
166      const accountTotals = await accounts.accountTotals();
167      const listeningTotals = await listening.totals();
168      const contentTotals = await content.contentTotals();
169      const votes = await d1Votes(env.DB).counts();
170      const [lobby, parties, stores] = await others;
171      return json(200, {
172        d1: {
173          ...accountTotals,
174          votes: { rows: votes.length, count: votes.reduce((a, v) => a + v.n, 0) },
175          ...listeningTotals,
176          content: contentTotals,
177        },
178        live: { lobby, parties },
179        kv: {
180          answerCache: stores.answerCache,
181          oauthGrants: stores.oauthGrants && {
182            grants: stores.oauthGrants.grants,
183            accounts: stores.oauthGrants.accounts,
184            complete: stores.oauthGrants.complete,
185          },
186        },
187        r2: { covers: stores.coverObjects },
188        countedAt: stores.at,
189        budgetPerDay: Number(env.JEV_DAILY_PER_USER),
190      });
191    }
192
193    case 'accounts': {
194      if (!id) {
195        const [list, stores] = await Promise.all([accounts.accountsPage(limit, offset), storeCounts(env)]);
196        return json(200, {
197          total: list.total,
198          accounts: list.accounts.map((a) => ({ ...a, apps: stores.oauthGrants?.byUser.get(a.id) ?? null })),
199        });
200      }
201      if (!ID.test(id)) return fail(404, 'no such account');
202      const view = await accounts.accountView(id);
203      if (!view) return fail(404, 'no such account');
204      const self = (await signedIn(request, accounts))?.id === id;
205      const [radio, takes, mine, publicSongs, budget, tabs, stores, apps] = await Promise.all([
206        d1Radio(env.DB).recent(id, KEPT),
207        d1Listening(env.DB).playerTakes(id, ACCOUNT_TAKES),
208        content.mine(id),
209        content.publicSongsBy(id),
210        orNull('budget', env.BUDGET.get(env.BUDGET.idFromName(id)).balance()),
211        orNull('tabs', env.TAB_HUB.get(env.TAB_HUB.idFromName(id)).sessions()),
212        storeCounts(env),
213        self ? orNull('apps', appsOf(servers(url.origin).as.getOAuthApi(env), id)) : Promise.resolve(null),
214      ]);
215      return json(200, {
216        ...view,
217        radio,
218        takes,
219        content: mine,
220        publicSongs,
221        budget,
222        openTabs: tabs?.length ?? null,
223        apps: {
224          count: apps ? apps.length : (stores.oauthGrants?.byUser.get(id) ?? (stores.oauthGrants?.complete ? 0 : null)),
225          // an account's own apps, by name, to itself only
226          ...(apps ? { yours: apps.map((a) => ({ app: a.app, scope: a.scope, createdAt: a.createdAt })) } : {}),
227        },
228      });
229    }
230
231    case 'votes': {
232      if (id) return fail(404, 'no such data');
233      const all = await d1Votes(env.DB).counts();
234      return json(200, { total: all.length, votes: all.slice(offset, offset + limit) });
235    }
236
237    case 'reactions':
238      if (id) return fail(404, 'no such data');
239      return json(200, await listening.reactionsPage(limit, offset, song));
240
241    case 'performances': {
242      if (!id) return json(200, await listening.performancesPage(limit, offset, song));
243      if (!ID.test(id)) return fail(404, 'no such performance');
244      const [one, segments] = await Promise.all([d1Listening(env.DB).performance(id), listening.segments(id)]);
245      if (!one) return fail(404, 'no such performance');
246      // the rows as stored, with the form's section of each; not `jevs`, the same answers again
247      const { jevs: _jevs, ...meta } = one;
248      return json(200, { ...meta, segments });
249    }
250
251    case 'songs': {
252      if (!id) return json(200, await content.songsPage(limit, offset));
253      if (!ID.test(id)) return fail(404, 'no such song');
254      const record = await content.songRecord(id);
255      return record ? json(200, record) : fail(404, 'no such song');
256    }
257
258    case 'covers':
259      if (id) return fail(404, 'no such data');
260      return json(200, await content.coversPage(limit, offset));
261    case 'comments':
262      if (id) return fail(404, 'no such data');
263      return json(200, await content.commentsPage(limit, offset));
264    case 'pitches':
265      if (id) return fail(404, 'no such data');
266      return json(200, await content.pitchesPage(limit, offset));
267    case 'jobs':
268      if (id) return fail(404, 'no such data');
269      return json(200, await content.jobsPage(limit, offset));
270    case 'activity': {
271      if (id) return fail(404, 'no such data');
272      const at = now();
273      const b = Number(url.searchParams.get('before') ?? at + 1);
274      const before = Number.isFinite(b) && b > 0 ? Math.min(b, at + 1) : at + 1;
275      const n = Number(url.searchParams.get('limit') ?? ACTIVITY_DEFAULT);
276      const count = Number.isInteger(n) && n > 0 ? Math.min(n, ACTIVITY_MAX) : ACTIVITY_DEFAULT;
277      return json(200, await activity(env.DB, before, count), { 'Cache-Control': `max-age=${ACTIVITY_CACHE_S}` });
278    }
279  }
280  return fail(404, 'no such data');
281}