zephyr/kaiyan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: zephyr:039013abb6b14fbc614b743139963bcb44f53863
Branches Tags
zephyr:main
...
compare: zephyr:2849b42745f27dd25987c18d0ca1433798c4e668
Branches Tags
zephyr:main

2 Commits
039013abb6 ... 2849b42745

Author SHA1 Message Date
Branden J Brown
2849b42745 kaiyan-index: get 7tv emote sets 2025-04-20 20:26:40 -04:00
Branden J Brown
e88d4b6cc0 emote: add 7tv model 2025-04-20 19:44:37 -04:00
4 changed files with 676 additions and 2 deletions
Show Stats Expand all files Collapse all files

50
cmd/kaiyan-index/main.go
View File

@ -5,8 +5,11 @@ import (
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"strconv"
"sync"
"time"
"github.com/twmb/franz-go/pkg/kgo"
@ -20,9 +23,15 @@ import (
func main() {
var (
dburi string
dburi string
streams []string
)
flag.StringVar(&dburi, "db", "", "SQLite database URI")
flag.Func("b", "Twitch broadcaster user ID", func(s string) error {
streams = append(streams, s)
_, err := strconv.Atoi(s)
return err
})
flag.Parse()
if flag.NArg() == 0 {
fail("provide broker addresses as args")
@ -30,6 +39,9 @@ func main() {
if dburi == "" {
fail("-db is mandatory")
}
if len(streams) == 0 {
fail("no broadcasters specified")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
go func() {
@ -37,6 +49,40 @@ func main() {
stop()
}()
parsers := make(map[string]emote.Parser)
{
var mu sync.Mutex
var wg sync.WaitGroup
wg.Add(len(streams))
for _, b := range streams {
go func() {
defer wg.Done()
req, err := http.NewRequestWithContext(ctx, "GET", "https://7tv.io/v3/users/twitch/"+b, nil)
if err != nil {
fail("%v", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
fail("%v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
slog.ErrorContext(ctx, "7tv", slog.String("broadcaster", b), slog.String("status", resp.Status))
return
}
em, err := emote.SevenTVv3(resp.Body)
if err != nil {
slog.ErrorContext(ctx, "7tv", slog.String("broadcaster", b), slog.Any("err", err))
return
}
mu.Lock()
parsers[b] = emote.NewParser(em...)
mu.Unlock()
}()
}
wg.Wait()
}
opts := sqlitex.PoolOptions{
Flags: sqlite.OpenCreate | sqlite.OpenReadWrite | sqlite.OpenURI,
PrepareConn: func(conn *sqlite.Conn) error {
@ -86,7 +132,7 @@ func main() {
var ems []emote.Emote
// TODO(branden): emote lists
for ctx.Err() == nil {
msgs, ems, err = batch(ctx, nil, q, st, msgs[:0], ems[:0])
msgs, ems, err = batch(ctx, parsers, q, st, msgs[:0], ems[:0])
if err != nil {
fail("%v", err)
}

44
emote/7tv.go Normal file
View File

@ -0,0 +1,44 @@
package emote
import (
"io"
"github.com/go-json-experiment/json"
)
// seventvUser has the fields of a 7TV user model which are relevant to Kaiyan.
type seventvUser struct {
EmoteSet emoteSet `json:"emote_set"`
}
type emoteSet struct {
Emotes []seventvEmote `json:"emotes"`
}
type seventvEmote struct {
ID string `json:"id"`
Name string `json:"name"`
// 7TV provides the CDN URL as well, but it's nested in further objects,
// not a complete URL, and it's also trivially derivable from the ID.
}
// SevenTVv3 parses the emotes from a 7TV v3/users/{platform}/{platform_id}
// response body.
func SevenTVv3(r io.Reader) ([]Emote, error) {
var u seventvUser
if err := json.UnmarshalRead(r, &u); err != nil {
return nil, err
}
ems := make([]Emote, 0, len(u.EmoteSet.Emotes))
for _, em := range u.EmoteSet.Emotes {
v := Emote{
ID: em.ID,
Name: em.Name,
Source: "7TV",
Link: "https://7tv.app/emotes/" + em.ID,
Image: "https://cdn.7tv.app/emote/" + em.ID + "/4x.webp",
}
ems = append(ems, v)
}
return ems, nil
}

46
emote/7tv_test.go Normal file
View File

@ -0,0 +1,46 @@
package emote_test
import (
_ "embed"
"strings"
"testing"
"git.sunturtle.xyz/zephyr/kaiyan/emote"
"github.com/google/go-cmp/cmp"
)
func TestSevenTVv3(t *testing.T) {
got, err := emote.SevenTVv3(strings.NewReader(seventvTwin))
if err != nil {
t.Error(err)
}
want := []emote.Emote{
{
ID: "01HBHC9CE00007KVA84SNERRE8",
Name: "powerSip",
Source: "7TV",
Link: "https://7tv.app/emotes/01HBHC9CE00007KVA84SNERRE8",
Image: "https://cdn.7tv.app/emote/01HBHC9CE00007KVA84SNERRE8/4x.webp",
},
{
ID: "01JDTNVJADW1R4YZB3F85VVT9T",
Name: "Jin",
Source: "7TV",
Link: "https://7tv.app/emotes/01JDTNVJADW1R4YZB3F85VVT9T",
Image: "https://cdn.7tv.app/emote/01JDTNVJADW1R4YZB3F85VVT9T/4x.webp",
},
{
ID: "01JE08SKFXM94FTG52GVMGFQZW",
Name: "IMissTwin",
Source: "7TV",
Link: "https://7tv.app/emotes/01JE08SKFXM94FTG52GVMGFQZW",
Image: "https://cdn.7tv.app/emote/01JE08SKFXM94FTG52GVMGFQZW/4x.webp",
},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("wrong emotes (+got/-want):\n%s", diff)
}
}
//go:embed testdata/twoinone1_.7tvv3.json
var seventvTwin string

538
emote/testdata/twoinone1_.7tvv3.json vendored Normal file
View File

@ -0,0 +1,538 @@
{
"id": "116723523",
"platform": "TWITCH",
"username": "twoinone1_",
"display_name": "twoinone1_",
"linked_at": 1730685664536,
"emote_capacity": 1000,
"emote_set_id": "01JD8YZCMPF628X8DZ7NAYFG8C",
"emote_set": {
"id": "01JD8YZCMPF628X8DZ7NAYFG8C",
"name": "twoinone1_'s Emotes",
"flags": 0,
"tags": [],
"immutable": false,
"privileged": false,
"emotes": [
{
"id": "01HBHC9CE00007KVA84SNERRE8",
"name": "powerSip",
"flags": 0,
"timestamp": 1696021656000,
"actor_id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"data": {
"id": "01HBHC9CE00007KVA84SNERRE8",
"name": "powerSip",
"flags": 0,
"tags": [
"csm",
"weeb",
"anime",
"sip"
],
"lifecycle": 3,
"state": [
"LISTED"
],
"listed": true,
"animated": false,
"owner": {
"id": "01FBXP96C0000APPS8GMQY5HFA",
"username": "kodyparbs",
"display_name": "Kodyparbs",
"avatar_url": "https://static-cdn.jtvnw.net/jtv_user_pictures/88453ec4-d53f-4413-a1c0-57319ccc679a-profile_image-300x300.png",
"style": {},
"role_ids": [
"01G68MMQFR0007J6GNM9E2M0TM"
],
"connections": [
{
"id": "417084954",
"platform": "TWITCH",
"username": "kodyparbs",
"display_name": "Kodyparbs",
"linked_at": 1627715312000,
"emote_capacity": 1000,
"emote_set_id": "01FBXP96C0000APPS8GMQY5HFA"
}
]
},
"host": {
"url": "//cdn.7tv.app/emote/01HBHC9CE00007KVA84SNERRE8",
"files": [
{
"name": "1x.webp",
"static_name": "1x.webp",
"width": 32,
"height": 32,
"frame_count": 1,
"size": 2568,
"format": "WEBP"
},
{
"name": "2x.webp",
"static_name": "2x.webp",
"width": 64,
"height": 64,
"frame_count": 1,
"size": 6388,
"format": "WEBP"
},
{
"name": "3x.webp",
"static_name": "3x.webp",
"width": 96,
"height": 96,
"frame_count": 1,
"size": 10816,
"format": "WEBP"
},
{
"name": "4x.webp",
"static_name": "4x.webp",
"width": 128,
"height": 128,
"frame_count": 1,
"size": 16062,
"format": "WEBP"
},
{
"name": "1x.avif",
"static_name": "1x.avif",
"width": 32,
"height": 32,
"frame_count": 1,
"size": 1632,
"format": "AVIF"
},
{
"name": "2x.avif",
"static_name": "2x.avif",
"width": 64,
"height": 64,
"frame_count": 1,
"size": 3607,
"format": "AVIF"
},
{
"name": "3x.avif",
"static_name": "3x.avif",
"width": 96,
"height": 96,
"frame_count": 1,
"size": 5813,
"format": "AVIF"
},
{
"name": "4x.avif",
"static_name": "4x.avif",
"width": 128,
"height": 128,
"frame_count": 1,
"size": 8011,
"format": "AVIF"
},
{
"name": "1x.png",
"static_name": "1x.png",
"width": 32,
"height": 32,
"frame_count": 1,
"size": 2919,
"format": "PNG"
},
{
"name": "2x.png",
"static_name": "2x.png",
"width": 64,
"height": 64,
"frame_count": 1,
"size": 8672,
"format": "PNG"
},
{
"name": "3x.png",
"static_name": "3x.png",
"width": 96,
"height": 96,
"frame_count": 1,
"size": 15712,
"format": "PNG"
},
{
"name": "4x.png",
"static_name": "4x.png",
"width": 128,
"height": 128,
"frame_count": 1,
"size": 23966,
"format": "PNG"
}
]
}
},
"origin_id": null
},
{
"id": "01JDTNVJADW1R4YZB3F85VVT9T",
"name": "Jin",
"flags": 0,
"timestamp": 1732840900941,
"actor_id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"data": {
"id": "01JDTNVJADW1R4YZB3F85VVT9T",
"name": "Jin",
"flags": 0,
"lifecycle": 3,
"state": [
"LISTED"
],
"listed": true,
"animated": false,
"owner": {
"id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"username": "twoinone1_",
"display_name": "twoinone1_",
"avatar_url": "https://static-cdn.jtvnw.net/jtv_user_pictures/3d606741-9960-484d-a484-1aa9a0f002b6-profile_image-300x300.png",
"style": {
"color": -5635841,
"paint_id": "01JQM68WYPTC7PFSE1QZKGWSTX"
},
"role_ids": [
"01F37R3RFR0000K96678WEQT01",
"01G68MMQFR0007J6GNM9E2M0TM"
],
"connections": [
{
"id": "116723523",
"platform": "TWITCH",
"username": "twoinone1_",
"display_name": "twoinone1_",
"linked_at": 1730685664536,
"emote_capacity": 1000,
"emote_set_id": "01JD8YZCMPF628X8DZ7NAYFG8C"
}
]
},
"host": {
"url": "//cdn.7tv.app/emote/01JDTNVJADW1R4YZB3F85VVT9T",
"files": [
{
"name": "1x.webp",
"static_name": "1x.webp",
"width": 74,
"height": 32,
"frame_count": 1,
"size": 774,
"format": "WEBP"
},
{
"name": "2x.webp",
"static_name": "2x.webp",
"width": 148,
"height": 64,
"frame_count": 1,
"size": 1768,
"format": "WEBP"
},
{
"name": "3x.webp",
"static_name": "3x.webp",
"width": 222,
"height": 96,
"frame_count": 1,
"size": 2880,
"format": "WEBP"
},
{
"name": "4x.webp",
"static_name": "4x.webp",
"width": 296,
"height": 128,
"frame_count": 1,
"size": 3916,
"format": "WEBP"
},
{
"name": "1x.avif",
"static_name": "1x.avif",
"width": 74,
"height": 32,
"frame_count": 1,
"size": 1192,
"format": "AVIF"
},
{
"name": "2x.avif",
"static_name": "2x.avif",
"width": 148,
"height": 64,
"frame_count": 1,
"size": 2598,
"format": "AVIF"
},
{
"name": "3x.avif",
"static_name": "3x.avif",
"width": 222,
"height": 96,
"frame_count": 1,
"size": 3963,
"format": "AVIF"
},
{
"name": "4x.avif",
"static_name": "4x.avif",
"width": 296,
"height": 128,
"frame_count": 1,
"size": 5406,
"format": "AVIF"
},
{
"name": "1x.png",
"static_name": "1x.png",
"width": 74,
"height": 32,
"frame_count": 1,
"size": 4913,
"format": "PNG"
},
{
"name": "2x.png",
"static_name": "2x.png",
"width": 148,
"height": 64,
"frame_count": 1,
"size": 13376,
"format": "PNG"
},
{
"name": "3x.png",
"static_name": "3x.png",
"width": 222,
"height": 96,
"frame_count": 1,
"size": 24123,
"format": "PNG"
},
{
"name": "4x.png",
"static_name": "4x.png",
"width": 296,
"height": 128,
"frame_count": 1,
"size": 34810,
"format": "PNG"
}
]
}
},
"origin_id": null
},
{
"id": "01JE08SKFXM94FTG52GVMGFQZW",
"name": "IMissTwin",
"flags": 0,
"timestamp": 1733028531709,
"actor_id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"data": {
"id": "01JE08SKFXM94FTG52GVMGFQZW",
"name": "IMissTwin",
"flags": 0,
"lifecycle": 3,
"state": [
"LISTED"
],
"listed": true,
"animated": false,
"owner": {
"id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"username": "twoinone1_",
"display_name": "twoinone1_",
"avatar_url": "https://static-cdn.jtvnw.net/jtv_user_pictures/3d606741-9960-484d-a484-1aa9a0f002b6-profile_image-300x300.png",
"style": {
"color": -5635841,
"paint_id": "01JQM68WYPTC7PFSE1QZKGWSTX"
},
"role_ids": [
"01F37R3RFR0000K96678WEQT01",
"01G68MMQFR0007J6GNM9E2M0TM"
],
"connections": [
{
"id": "116723523",
"platform": "TWITCH",
"username": "twoinone1_",
"display_name": "twoinone1_",
"linked_at": 1730685664536,
"emote_capacity": 1000,
"emote_set_id": "01JD8YZCMPF628X8DZ7NAYFG8C"
}
]
},
"host": {
"url": "//cdn.7tv.app/emote/01JE08SKFXM94FTG52GVMGFQZW",
"files": [
{
"name": "1x.webp",
"static_name": "1x.webp",
"width": 44,
"height": 32,
"frame_count": 1,
"size": 748,
"format": "WEBP"
},
{
"name": "2x.webp",
"static_name": "2x.webp",
"width": 88,
"height": 64,
"frame_count": 1,
"size": 1964,
"format": "WEBP"
},
{
"name": "3x.webp",
"static_name": "3x.webp",
"width": 132,
"height": 96,
"frame_count": 1,
"size": 3444,
"format": "WEBP"
},
{
"name": "4x.webp",
"static_name": "4x.webp",
"width": 176,
"height": 128,
"frame_count": 1,
"size": 5210,
"format": "WEBP"
},
{
"name": "1x.avif",
"static_name": "1x.avif",
"width": 44,
"height": 32,
"frame_count": 1,
"size": 1408,
"format": "AVIF"
},
{
"name": "2x.avif",
"static_name": "2x.avif",
"width": 88,
"height": 64,
"frame_count": 1,
"size": 3291,
"format": "AVIF"
},
{
"name": "3x.avif",
"static_name": "3x.avif",
"width": 132,
"height": 96,
"frame_count": 1,
"size": 5577,
"format": "AVIF"
},
{
"name": "4x.avif",
"static_name": "4x.avif",
"width": 176,
"height": 128,
"frame_count": 1,
"size": 8071,
"format": "AVIF"
},
{
"name": "1x.png",
"static_name": "1x.png",
"width": 44,
"height": 32,
"frame_count": 1,
"size": 4904,
"format": "PNG"
},
{
"name": "2x.png",
"static_name": "2x.png",
"width": 88,
"height": 64,
"frame_count": 1,
"size": 16973,
"format": "PNG"
},
{
"name": "3x.png",
"static_name": "3x.png",
"width": 132,
"height": 96,
"frame_count": 1,
"size": 34591,
"format": "PNG"
},
{
"name": "4x.png",
"static_name": "4x.png",
"width": 176,
"height": 128,
"frame_count": 1,
"size": 56643,
"format": "PNG"
}
]
}
},
"origin_id": null
}
],
"emote_count": 3,
"capacity": 1000,
"owner": null
},
"user": {
"id": "01JBTEEZ8R6A1J8RMZ581A1V02",
"username": "twoinone1_",
"display_name": "twoinone1_",
"created_at": 1730685664536,
"avatar_url": "https://static-cdn.jtvnw.net/jtv_user_pictures/3d606741-9960-484d-a484-1aa9a0f002b6-profile_image-300x300.png",
"style": {
"color": -5635841,
"paint_id": "01JQM68WYPTC7PFSE1QZKGWSTX"
},
"emote_sets": [
{
"id": "01JD8YZCMPF628X8DZ7NAYFG8C",
"name": "twoinone1_'s Emotes",
"flags": 0,
"tags": [],
"capacity": 1000
},
{
"id": "01JR4JFB07TJHTFCCF2BDZM8MH",
"name": "Personal Emote Set",
"flags": 4,
"tags": [],
"capacity": 5
}
],
"roles": [
"01G68MMQFR0007J6GNM9E2M0TM",
"01F37R3RFR0000K96678WEQT01"
],
"connections": [
{
"id": "116723523",
"platform": "TWITCH",
"username": "twoinone1_",
"display_name": "twoinone1_",
"linked_at": 1730685664536,
"emote_capacity": 1000,
"emote_set_id": "01JD8YZCMPF628X8DZ7NAYFG8C",
"emote_set": null
}
]
}
}