emote/sqlitestore: implement index in terms of sqlite

emote/sqlitestore/metrics.sql

@@ -0,0 +1,12 @@
+SELECT
+	id,
+	name, -- name, source, &c. are constant per id, and sqlite lets us select without aggregate
+	source,
+	link,
+	image,
+	COUNT(*) AS tokens,
+	COUNT(DISTINCT message) AS messages,
+	COUNT(DISTINCT sender) AS senders
+FROM emote
+WHERE channel = :channel AND time BETWEEN :start AND :end
+GROUP BY id

emote/sqlitestore/schema.sql

@@ -0,0 +1,17 @@
+PRAGMA journal_mode = WAL;
+
+CREATE TABLE IF NOT EXISTS emote (
+	channel TEXT NOT NULL,
+	message TEXT NOT NULL,
+	time INTEGER NOT NULL,
+	sender TEXT NOT NULL,
+	id TEXT NOT NULL,
+	-- The following columns could be normalized in a separate table,
+	-- but they're here for simplicity.
+	name TEXT NOT NULL,
+	source TEXT NOT NULL,
+	link TEXT NOT NULL,
+	image TEXT NOT NULL
+) STRICT;
+
+CREATE INDEX IF NOT EXISTS channel_time ON emote (channel, time);

emote/sqlitestore/store.go

@@ -0,0 +1,114 @@
+package sqlitestore
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"time"
+
+	"zombiezen.com/go/sqlite/sqlitex"
+
+	"git.sunturtle.xyz/zephyr/kaiyan/emote"
+)
+
+type Store struct {
+	pool *sqlitex.Pool
+}
+
+//go:embed schema.sql
+var schemaSQL string
+
+//go:embed metrics.sql
+var metricsSQL string
+
+// Open creates a metrics store using db as its underlying storage.
+// It initializes the schema used for Kaiyan.
+func Open(ctx context.Context, db *sqlitex.Pool) (*Store, error) {
+	conn, err := db.Take(ctx)
+	defer db.Put(conn)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get connection from pool: %w", err)
+	}
+	if err := sqlitex.ExecuteScript(conn, schemaSQL, nil); err != nil {
+		return nil, fmt.Errorf("couldn't run migration: %w", err)
+	}
+	st := Store{db}
+	return &st, nil
+}
+
+// Close closes the database.
+func (db *Store) Close() error {
+	return db.pool.Close()
+}
+
+// Record stores the emotes in a given message.
+func (db *Store) Record(ctx context.Context, channel, message, sender string, tm time.Time, emotes []emote.Emote) (err error) {
+	conn, err := db.pool.Take(ctx)
+	defer db.pool.Put(conn)
+	if err != nil {
+		return fmt.Errorf("couldn't get connection to record emotes: %w", err)
+	}
+	defer sqlitex.Transaction(conn)(&err)
+
+	st, err := conn.Prepare(`INSERT INTO emote (channel, message, time, sender, id, name, source, link, image) VALUES (:channel, :message, :time, :sender, :id, :name, :source, :link, :image)`)
+	if err != nil {
+		return fmt.Errorf("couldn't prepare emote insert: %w", err)
+	}
+	st.SetText(":channel", channel)
+	st.SetText(":message", message)
+	st.SetInt64(":time", tm.UnixNano())
+	st.SetText(":sender", sender)
+	for _, em := range emotes {
+		st.SetText(":id", em.ID)
+		st.SetText(":name", em.Name)
+		st.SetText(":source", em.Source)
+		st.SetText(":link", em.Link)
+		st.SetText(":image", em.Image)
+		_, err := st.Step()
+		if err != nil {
+			return fmt.Errorf("couldn't insert emote: %w", err)
+		}
+		st.Reset() // NOTE(branden): bound parameters are retained
+	}
+	return nil
+}
+
+// Metrics appends emote usage information for a channel in a given time range.
+func (db *Store) Metrics(ctx context.Context, channel string, start, end time.Time, onto []emote.Metric) ([]emote.Metric, error) {
+	conn, err := db.pool.Take(ctx)
+	defer db.pool.Put(conn)
+	if err != nil {
+		return onto, fmt.Errorf("couldn't get connection for metrics: %w", err)
+	}
+	st, err := conn.Prepare(metricsSQL)
+	if err != nil {
+		return onto, fmt.Errorf("couldn't prepare metrics statement: %w", err)
+	}
+	st.SetText(":channel", channel)
+	st.SetInt64(":start", start.UnixNano())
+	st.SetInt64(":end", end.UnixNano())
+	for {
+		ok, err := st.Step()
+		if err != nil {
+			return onto, fmt.Errorf("couldn't step metrics selection: %w", err)
+		}
+		if !ok {
+			break
+		}
+
+		m := emote.Metric{
+			Emote: emote.Emote{
+				ID:     st.ColumnText(0),
+				Name:   st.ColumnText(1),
+				Source: st.ColumnText(2),
+				Link:   st.ColumnText(3),
+				Image:  st.ColumnText(4),
+			},
+			Tokens:   st.ColumnInt64(5),
+			Messages: st.ColumnInt64(6),
+			Users:    st.ColumnInt64(7),
+		}
+		onto = append(onto, m)
+	}
+	return onto, st.Reset()
+}

emote/sqlitestore/store_test.go

@@ -0,0 +1,141 @@
+package sqlitestore_test
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"git.sunturtle.xyz/zephyr/kaiyan/emote"
+	"git.sunturtle.xyz/zephyr/kaiyan/emote/sqlitestore"
+	"github.com/google/go-cmp/cmp"
+	"zombiezen.com/go/sqlite"
+	"zombiezen.com/go/sqlite/sqlitex"
+)
+
+func testDB(name string) *sqlitex.Pool {
+	opts := sqlitex.PoolOptions{
+		Flags: sqlite.OpenCreate | sqlite.OpenReadWrite | sqlite.OpenMemory | sqlite.OpenSharedCache | sqlite.OpenURI,
+	}
+	pool, err := sqlitex.NewPool(fmt.Sprintf("file:%s.db?mode=memory&cache=shared", name), opts)
+	if err != nil {
+		panic(err)
+	}
+	return pool
+}
+
+func TestStore(t *testing.T) {
+	ems := []struct {
+		channel, message, sender string
+		ts                       int64
+		emotes                   []emote.Emote
+	}{
+		{
+			channel: "kessoku",
+			message: "1",
+			sender:  "ryo",
+			ts:      1000,
+			emotes: []emote.Emote{
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+			},
+		},
+		{
+			channel: "kessoku",
+			message: "2",
+			sender:  "seika",
+			ts:      2000,
+			emotes: []emote.Emote{
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+			},
+		},
+		{
+			channel: "sickhack",
+			message: "3",
+			sender:  "ryo",
+			ts:      1500,
+			emotes: []emote.Emote{
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+				{ID: "nijika", Name: "nijika"},
+				{ID: "kita", Name: "kita"},
+			},
+		},
+	}
+	st, err := sqlitestore.Open(t.Context(), testDB("TestStore"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range ems {
+		err := st.Record(t.Context(), m.channel, m.message, m.sender, time.Unix(m.ts, 0), m.emotes)
+		if err != nil {
+			t.Errorf("couldn't record message %s: %v", m.message, err)
+		}
+	}
+	cases := []struct {
+		name       string
+		channel    string
+		start, end int64
+		want       []emote.Metric
+	}{
+		{
+			name:    "channel",
+			channel: "kessoku",
+			start:   0,
+			end:     1e6,
+			want: []emote.Metric{
+				{
+					Emote:    emote.Emote{ID: "kita", Name: "kita"},
+					Tokens:   4,
+					Messages: 2,
+					Users:    2,
+				},
+				{
+					Emote:    emote.Emote{ID: "nijika", Name: "nijika"},
+					Tokens:   4,
+					Messages: 2,
+					Users:    2,
+				},
+			},
+		},
+		{
+			name:    "time",
+			channel: "kessoku",
+			start:   1250,
+			end:     1e6,
+			want: []emote.Metric{
+				{
+					Emote:    emote.Emote{ID: "kita", Name: "kita"},
+					Tokens:   2,
+					Messages: 1,
+					Users:    1,
+				},
+				{
+					Emote:    emote.Emote{ID: "nijika", Name: "nijika"},
+					Tokens:   2,
+					Messages: 1,
+					Users:    1,
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		// Don't use Run because we have earlier t.Fails.
+		got, err := st.Metrics(t.Context(), c.channel, time.Unix(c.start, 0), time.Unix(c.end, 0), nil)
+		if err != nil {
+			t.Error(err)
+		}
+		// Sort results by emote ID so we can easily diff.
+		// In particular, we don't expect the store to sort for us.
+		slices.SortFunc(got, func(a, b emote.Metric) int { return strings.Compare(a.Emote.ID, b.Emote.ID) })
+		if diff := cmp.Diff(c.want, got); diff != "" {
+			t.Errorf("wrong results (+got/-want):\n%s", diff)
+		}
+	}
+}

emote/store.go

@@ -0,0 +1,26 @@
+package emote
+
+import (
+	"context"
+	"time"
+)
+
+// Store is a store of emote usage records.
+type Store interface {
+	// Record stores the emotes in a given message.
+	Record(ctx context.Context, channel, message, sender string, tm time.Time, emotes []Emote) error
+	// Metrics appends emote usage information for a channel in a given time range.
+	Metrics(ctx context.Context, channel string, start, end time.Time, onto []Metric) ([]Metric, error)
+}
+
+// Metric is the metrics for a single emote.
+type Metric struct {
+	// Emote is the emote that the metrics describe.
+	Emote Emote
+	// Tokens is the total number of occurrences of the emote.
+	Tokens int64
+	// Messages is the number of unique messages which contained the emote.
+	Messages int64
+	// Users is the number of users who used the emote.
+	Users int64
+}

go.mod

@@ -6,10 +6,21 @@ require (
 	github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874
 	github.com/google/go-cmp v0.7.0
 	github.com/twmb/franz-go v1.18.1
+	zombiezen.com/go/sqlite v1.4.0
 )
 
 require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/twmb/franz-go/pkg/kmsg v1.9.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	modernc.org/libc v1.55.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.33.1 // indirect
 )

go.sum

@@ -1,14 +1,59 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874 h1:F8d1AJ6M9UQCavhwmO6ZsrYLfG8zVFWfEfMS2MXPkSY=
 github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
 github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc=
 github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M=
 github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M=
 github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg=
 golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
 golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
+modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
+modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
+modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
+modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
+modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
+modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
+modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
+zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=