cmd/racedb: program to load races into an sqlite db

cmd/racedb/db.go

@@ -0,0 +1,325 @@
+package main
+
+import (
+	"cmp"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"zombiezen.com/go/sqlite"
+	"zombiezen.com/go/sqlite/sqlitemigration"
+	"zombiezen.com/go/sqlite/sqlitex"
+
+	"git.sunturtle.xyz/zephyr/horse/umadump/replay"
+)
+
+const appID int32 = 0x4acedb
+
+//go:embed schema-00.sql
+var schema0 string
+
+//go:embed schema-final.sql
+var schemaFinal string
+
+func opendb(uri, journalMode string) *sqlitemigration.Pool {
+	schema := sqlitemigration.Schema{
+		Migrations: []string{
+			schema0,
+		},
+		AppID:               appID,
+		RepeatableMigration: schemaFinal,
+	}
+	j := "PRAGMA journal_mode=" + journalMode
+	opts := sqlitemigration.Options{
+		Flags: sqlite.OpenReadWrite | sqlite.OpenCreate | sqlite.OpenURI,
+		PrepareConn: func(conn *sqlite.Conn) error {
+			if journalMode == "" {
+				return nil
+			}
+			stmt, _, err := conn.PrepareTransient(j)
+			if err != nil {
+				return err
+			}
+			_, err = stmt.Step()
+			return errors.Join(err, stmt.Finalize())
+		},
+	}
+	db := sqlitemigration.NewPool(uri, schema, opts)
+	return db
+}
+
+func insert(ctx context.Context, db *sqlitemigration.Pool, log *slog.Logger, race *replay.Race, simdata *replay.RaceSimulateData) (err error) {
+	conn, err := db.Take(ctx)
+	if err != nil {
+		return err
+	}
+	defer db.Put(conn)
+
+	defer sqlitex.Transaction(conn)(&err)
+
+	log.DebugContext(ctx, "horses",
+		slog.Int("num", int(race.NumRaceHorses)),
+		slog.Int("actual", len(race.RaceHorse)),
+	)
+	var trainedCharas [18]int64
+	for i := range race.RaceHorse {
+		c := &race.RaceHorse[i]
+		id, err := insertTrainedChara(conn, c)
+		if err != nil {
+			return err
+		}
+		trainedCharas[i] = id
+		if err := insertTrainedCharaSkill(conn, c, id); err != nil {
+			return err
+		}
+		log.DebugContext(ctx, "horse",
+			slog.String("trainer", c.TrainerName),
+			slog.Int("trained", int(c.ResponseHorseData.TrainedCharaID)),
+			slog.Int("card", int(c.ResponseHorseData.CardID)),
+			slog.Int64("id", id),
+		)
+	}
+	if err = insertRaceCourseSet(conn, race); err != nil {
+		return err
+	}
+	log.DebugContext(ctx, "course", slog.Int("id", int(race.RaceCourseSet.ID)))
+	configID, err := insertRaceConfig(conn, race)
+	if err != nil {
+		return err
+	}
+	log.DebugContext(ctx, "config", slog.Int64("id", configID))
+	raceID, err := insertRace(conn, race, simdata, &trainedCharas, configID)
+	if err != nil {
+		code := sqlite.ErrCode(err)
+		if code == sqlite.ResultConstraintUnique {
+			log.InfoContext(ctx, "already loaded")
+			return nil
+		}
+		return err
+	}
+	log.DebugContext(ctx, "race", slog.Int64("id", raceID))
+	err = errors.Join(
+		insertRaceHorses(ctx, log, conn, race, simdata, &trainedCharas, raceID),
+		insertRaceFrames(ctx, log, conn, simdata, raceID),
+		insertRaceEvents(ctx, log, conn, simdata, raceID),
+	)
+	return err
+}
+
+//go:embed insert-trained_chara.sql
+var insertTrainedCharaSQL string
+
+func insertTrainedChara(conn *sqlite.Conn, c *replay.HorseData) (int64, error) {
+	stmt := conn.Prep(insertTrainedCharaSQL)
+	trainer := cmp.Or(c.ResponseHorseData.ViewerID, c.ResponseHorseData.OwnerViewerID)
+	stmt.SetText("$trainer", c.TrainerName)
+	stmt.SetInt64("$trainer_id", trainer)
+	stmt.SetInt64("$trained_chara_id", int64(c.ResponseHorseData.TrainedCharaID))
+	stmt.SetInt64("$chara_id", int64(c.CharaID))
+	stmt.SetText("$chara_name", c.CharaName)
+	stmt.SetInt64("$card_id", int64(c.ResponseHorseData.CardID))
+	stmt.SetInt64("$rarity", int64(c.Rarity))
+	stmt.SetInt64("$mob_id", int64(c.MobID))
+	stmt.SetInt64("$raw_speed", int64(c.ResponseHorseData.Speed))
+	stmt.SetInt64("$raw_stamina", int64(c.ResponseHorseData.Stamina))
+	stmt.SetInt64("$raw_power", int64(c.ResponseHorseData.Pow))
+	stmt.SetInt64("$raw_guts", int64(c.ResponseHorseData.Guts))
+	stmt.SetInt64("$raw_wit", int64(c.ResponseHorseData.Wiz))
+	stmt.SetInt64("$turf", int64(c.ResponseHorseData.ProperGroundTurf))
+	stmt.SetInt64("$dirt", int64(c.ResponseHorseData.ProperGroundDirt))
+	stmt.SetInt64("$sprint", int64(c.ResponseHorseData.ProperDistanceShort))
+	stmt.SetInt64("$mile", int64(c.ResponseHorseData.ProperDistanceMile))
+	stmt.SetInt64("$medium", int64(c.ResponseHorseData.ProperDistanceMiddle))
+	stmt.SetInt64("$long", int64(c.ResponseHorseData.ProperDistanceLong))
+	stmt.SetInt64("$front_runner", int64(c.ResponseHorseData.ProperRunningStyleNige))
+	stmt.SetInt64("$pace_chaser", int64(c.ResponseHorseData.ProperRunningStyleSenko))
+	stmt.SetInt64("$late_surger", int64(c.ResponseHorseData.ProperRunningStyleSashi))
+	stmt.SetInt64("$end_closer", int64(c.ResponseHorseData.ProperRunningStyleOikomi))
+	stmt.SetInt64("$fans", int64(c.TrainedCharaData.Fans))
+	stmt.SetInt64("$rank", int64(c.TrainedCharaData.Rank))
+	stmt.SetInt64("$rank_score", int64(c.TrainedCharaData.RankScore))
+	r, err := sqlitex.ResultInt64(stmt)
+	err = errors.Join(err, stmt.Reset())
+	if err != nil {
+		err = fmt.Errorf("inserting trained chara: %w", err)
+	}
+	return r, err
+}
+
+func insertTrainedCharaSkill(conn *sqlite.Conn, c *replay.HorseData, id int64) error {
+	const sql = `INSERT OR IGNORE INTO trained_chara_skill VALUES (?, ?, ?)`
+	stmt := conn.Prep(sql)
+	stmt.BindInt64(1, id)
+	for _, sk := range c.ResponseHorseData.SkillArray {
+		stmt.BindInt64(2, int64(sk.SkillID))
+		stmt.BindInt64(3, int64(sk.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting trained chara id %d skill %d: %w", id, sk.SkillID, err)
+		}
+	}
+	return nil
+}
+
+func insertRaceCourseSet(conn *sqlite.Conn, race *replay.Race) error {
+	const sql = `INSERT OR IGNORE INTO race_course_set VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	stmt := conn.Prep(sql)
+	stmt.BindInt64(1, int64(race.RaceCourseSet.ID))
+	stmt.BindInt64(2, int64(race.RaceCourseSet.RaceTrackID))
+	stmt.BindInt64(3, int64(race.RaceCourseSet.Distance))
+	stmt.BindInt64(4, int64(race.RaceCourseSet.Ground))
+	stmt.BindInt64(5, int64(race.RaceCourseSet.Inout))
+	stmt.BindInt64(6, int64(race.RaceCourseSet.Turn))
+	stmt.BindInt64(7, int64(race.RaceCourseSet.FenceSet))
+	stmt.BindInt64(8, int64(race.RaceCourseSet.FloatLaneMax))
+	stmt.BindInt64(9, int64(race.RaceCourseSet.CourseSetStatusID))
+	stmt.BindInt64(10, int64(race.RaceCourseSet.FinishTimeMin))
+	stmt.BindInt64(11, int64(race.RaceCourseSet.FinishTimeMinRandomRange))
+	stmt.BindInt64(12, int64(race.RaceCourseSet.FinishTimeMax))
+	stmt.BindInt64(13, int64(race.RaceCourseSet.FinishTimeMaxRandomRange))
+	stmt.BindFloat(14, float64(race.CourseSectionDistance))
+	stmt.BindInt64(15, int64(race.CourseDistanceType))
+	stmt.BindInt64(16, int64(race.CourseFurlongNum))
+	stmt.BindFloat(17, float64(race.BaseSpeed))
+	_, err := stmt.Step()
+	err = errors.Join(err, stmt.Reset())
+	if err != nil {
+		return fmt.Errorf("inserting race course: %w", err)
+	}
+	return nil
+}
+
+//go:embed insert-race_config.sql
+var insertRaceConfigSQL string
+
+func insertRaceConfig(conn *sqlite.Conn, race *replay.Race) (int64, error) {
+	stmt := conn.Prep(insertRaceConfigSQL)
+	stmt.SetInt64("$type", int64(race.RaceType))
+	stmt.SetInt64("$course_set_id", int64(race.RaceCourseSet.ID))
+	stmt.SetInt64("$ground_condition", int64(race.GroundCondition))
+	stmt.SetInt64("$weather", int64(race.Weather))
+	stmt.SetInt64("$season", int64(race.Season))
+	stmt.SetInt64("$time", int64(race.Time))
+	stmt.SetInt64("$num_horse", int64(race.NumRaceHorses))
+	r, err := sqlitex.ResultInt64(stmt)
+	err = errors.Join(err, stmt.Reset())
+	if err != nil {
+		err = fmt.Errorf("inserting race config: %w", err)
+	}
+	return r, err
+}
+
+//go:embed insert-race.sql
+var insertRaceSQL string
+
+func insertRace(conn *sqlite.Conn, race *replay.Race, simdata *replay.RaceSimulateData, trainedCharas *[18]int64, configID int64) (int64, error) {
+	stmt := conn.Prep(insertRaceSQL)
+	stmt.SetInt64("$config_id", configID)
+	stmt.SetInt64("$seed", int64(race.RandomSeed))
+	stmt.SetFloat("$distance_diff_max", float64(simdata.DistanceDiffMax))
+	stmt.SetInt64("$frame_count", int64(simdata.FrameCount))
+	// zombiezen/sqlite doesn't seem to expose an API to find the index of a
+	// named parameter, so even though we were using names before, we switch
+	// to indices for trained chara IDs.
+	for i, id := range trainedCharas {
+		stmt.BindInt64(5+i, id)
+	}
+	r, err := sqlitex.ResultInt64(stmt)
+	err = errors.Join(err, stmt.Reset())
+	if err != nil {
+		err = fmt.Errorf("inserting race: %w", err)
+	}
+	return r, err
+}
+
+//go:embed insert-race_horse.sql
+var insertRaceHorseSQL string
+
+func insertRaceHorses(ctx context.Context, log *slog.Logger, conn *sqlite.Conn, race *replay.Race, simdata *replay.RaceSimulateData, trainedCharas *[18]int64, raceID int64) error {
+	stmt := conn.Prep(insertRaceHorseSQL)
+	stmt.SetInt64("$race", raceID)
+	for i, c := range race.RaceHorse {
+		log.DebugContext(ctx, "race horse", slog.Int64("trained", trainedCharas[i]), slog.Int("index", int(c.HorseIndex)))
+		stmt.SetInt64("$horse", trainedCharas[i])
+		stmt.SetInt64("$horse_index", int64(c.HorseIndex))
+		stmt.SetInt64("$post", int64(c.PostNumber))
+		stmt.SetInt64("$finish_order", int64(c.FinishOrder))
+		stmt.SetFloat("$finish_time_raw", float64(c.FinishTimeRaw))
+		stmt.SetFloat("$finish_time_scaled", float64(c.FinishTimeScaled))
+		stmt.SetFloat("$finish_diff_time_from_prev", float64(c.FinishDiffTimeFromPrev))
+		stmt.SetInt64("$running_style", int64(c.ResponseHorseData.RunningStyle))
+		stmt.SetInt64("$running_style_ex", int64(c.RunningStyleEx))
+		stmt.SetInt64("$active_distance_aptitude", int64(c.ActiveProperDistance))
+		stmt.SetInt64("$active_surface_aptitude", int64(c.ActiveProperGroundType))
+		stmt.SetInt64("$mood", int64(c.RaceParam.Motivation))
+		stmt.SetFloat("$mood_coef", float64(c.RaceParam.MotivationCoef))
+		stmt.SetFloat("$base_speed", float64(c.RaceParam.BaseSpeed))
+		stmt.SetFloat("$base_stamina", float64(c.RaceParam.BaseStamina))
+		stmt.SetFloat("$base_power", float64(c.RaceParam.BasePow))
+		stmt.SetFloat("$base_guts", float64(c.RaceParam.BaseGuts))
+		stmt.SetFloat("$base_wit", float64(c.RaceParam.BaseWiz))
+		stmt.SetInt64("$popularity", int64(c.Popularity))
+		stmt.SetInt64("$popularity_rank_left", int64(c.PopularityRankLeft))
+		stmt.SetInt64("$popularity_rank_center", int64(c.PopularityRankCenter))
+		stmt.SetInt64("$popularity_rank_right", int64(c.PopularityRankRight))
+		stmt.SetFloat("$start_delay", float64(simdata.HorseResult[i].StartDelayTime))
+		stmt.SetFloat("$spurt_start_distance", float64(simdata.HorseResult[i].LastSpurtStartDistance))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting race horses: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertRaceFrames(ctx context.Context, log *slog.Logger, conn *sqlite.Conn, simdata *replay.RaceSimulateData, raceID int64) error {
+	const sql = `INSERT INTO race_frame VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	stmt := conn.Prep(sql)
+	stmt.BindInt64(1, raceID)
+	for _, f := range simdata.Frame {
+		log.DebugContext(ctx, "frame", slog.Float64("time", float64(f.Time)))
+		stmt.BindFloat(2, float64(f.Time))
+		for i, h := range f.HorseFrame {
+			stmt.BindInt64(3, int64(i))
+			stmt.BindFloat(4, float64(h.Distance))
+			stmt.BindInt64(5, int64(h.LanePosition))
+			stmt.BindInt64(6, int64(h.Speed))
+			stmt.BindInt64(7, int64(h.HP))
+			stmt.BindInt64(8, int64(h.TemptationMode))
+			stmt.BindInt64(9, int64(h.BlockFrontHorseIndex))
+			_, err := stmt.Step()
+			err = errors.Join(err, stmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting race frames: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func insertRaceEvents(ctx context.Context, log *slog.Logger, conn *sqlite.Conn, simdata *replay.RaceSimulateData, raceID int64) error {
+	const sql = `INSERT INTO race_event VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	stmt := conn.Prep(sql)
+	for _, e := range simdata.Event {
+		log.DebugContext(ctx, "event", slog.Float64("time", float64(e.Event.FrameTime)), slog.Int("type", int(e.Event.Type)))
+		// Easier to clear all bindings every time now
+		// than to bind nulls to all the parameters that need them below.
+		stmt.ClearBindings()
+		stmt.BindInt64(1, raceID)
+		stmt.BindFloat(2, float64(e.Event.FrameTime))
+		stmt.BindInt64(3, int64(e.Event.Type))
+		for i, p := range e.Event.Param {
+			stmt.BindInt64(4+i, int64(p))
+		}
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting race events: %w", err)
+		}
+	}
+	return nil
+}

cmd/racedb/insert-race.sql

@@ -0,0 +1,49 @@
+INSERT OR ROLLBACK INTO race (
+	config_id,
+	seed,
+	distance_diff_max,
+	frame_count,
+	horse_1,
+	horse_2,
+	horse_3,
+	horse_4,
+	horse_5,
+	horse_6,
+	horse_7,
+	horse_8,
+	horse_9,
+	horse_10,
+	horse_11,
+	horse_12,
+	horse_13,
+	horse_14,
+	horse_15,
+	horse_16,
+	horse_17,
+	horse_18
+)
+VALUES (
+	$config_id,
+	$seed,
+	$distance_diff_max,
+	$frame_count,
+	$horse_1,
+	$horse_2,
+	$horse_3,
+	$horse_4,
+	$horse_5,
+	$horse_6,
+	$horse_7,
+	$horse_8,
+	$horse_9,
+	$horse_10,
+	$horse_11,
+	$horse_12,
+	$horse_13,
+	$horse_14,
+	$horse_15,
+	$horse_16,
+	$horse_17,
+	$horse_18
+)
+RETURNING id

cmd/racedb/insert-race_config.sql

@@ -0,0 +1,4 @@
+INSERT INTO race_config (type, course_set_id, ground_condition, weather, season, time, num_horse)
+VALUES ($type, $course_set_id, $ground_condition, $weather, $season, $time, $num_horse)
+ON CONFLICT DO UPDATE SET id=id
+RETURNING id

cmd/racedb/insert-race_horse.sql

@@ -0,0 +1,12 @@
+INSERT INTO race_horse
+VALUES (
+	$race, $horse,
+	$horse_index, $post,
+	$finish_order, $finish_time_raw, $finish_time_scaled, $finish_diff_time_from_prev,
+	$running_style, $running_style_ex,
+	$active_distance_aptitude, $active_surface_aptitude,
+	$mood, $mood_coef,
+	$base_speed, $base_stamina, $base_power, $base_guts, $base_wit,
+	$popularity, $popularity_rank_left, $popularity_rank_center, $popularity_rank_right,
+	$start_delay, $spurt_start_distance
+)

cmd/racedb/insert-trained_chara.sql

@@ -0,0 +1,19 @@
+INSERT INTO trained_chara (
+	trainer, trainer_id, trained_chara_id,
+	chara_id, chara_name, card_id, rarity, mob_id,
+	raw_speed, raw_stamina, raw_power, raw_guts, raw_wit,
+	turf, dirt,
+	sprint, mile, medium, long,
+	front_runner, pace_chaser, late_surger, end_closer,
+	fans, rank, rank_score
+) VALUES (
+	$trainer, $trainer_id, $trained_chara_id,
+	$chara_id, $chara_name, $card_id, $rarity, $mob_id,
+	$raw_speed, $raw_stamina, $raw_power, $raw_guts, $raw_wit,
+	$turf, $dirt,
+	$sprint, $mile, $medium, $long,
+	$front_runner, $pace_chaser, $late_surger, $end_closer,
+	$fans, $rank, $rank_score
+)
+ON CONFLICT DO UPDATE SET id=id
+RETURNING id

cmd/racedb/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"path/filepath"
+
+	"git.sunturtle.xyz/zephyr/horse/umadump/replay"
+)
+
+func main() {
+	var (
+		uri, journalMode string
+		fs               []string
+	)
+	flag.StringVar(&uri, "db", "race.sqlite3", "output database `URI` or filename")
+	flag.StringVar(&journalMode, "journal", "memory", "journal `mode` to apply to database connections")
+	flag.Func("d", "`dir`ectory containing race dumps to load (repeatable)", func(s string) error {
+		es, err := os.ReadDir(s)
+		for _, e := range es {
+			if e.IsDir() {
+				continue
+			}
+			fs = append(fs, filepath.Join(s, e.Name()))
+		}
+		return err
+	})
+	flag.Parse()
+	fs = append(fs, flag.Args()...)
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	go func() {
+		<-ctx.Done()
+		stop()
+	}()
+
+	db := opendb(uri, journalMode)
+	ok := 0
+	for _, n := range fs {
+		log := slog.With(slog.String("file", n))
+		log.Info("begin")
+		f, err := os.Open(n)
+		if err != nil {
+			log.Error("open", slog.Any("err", err))
+			continue
+		}
+		var race replay.Race
+		d := json.NewDecoder(f)
+		d.DisallowUnknownFields()
+		log.Debug("decode")
+		if err := d.Decode(&race); err != nil {
+			log.Error("load", slog.Any("err", err))
+			continue
+		}
+		log.Debug("simdata")
+		simdata, err := replay.DecodeSim(race.SimDataBase64)
+		if err != nil {
+			log.Error("simdata", slog.Any("err", err))
+			continue
+		}
+		log.Debug("insert")
+		err = insert(ctx, db, log, &race, simdata)
+		if err != nil {
+			log.Error("insert", slog.Any("err", err))
+			continue
+		}
+		ok++
+	}
+	slog.Info("done", slog.Int("races", ok))
+}

cmd/racedb/schema-00.sql

@@ -0,0 +1,234 @@
+-- trained_chara lists horses who appear in races.
+CREATE TABLE IF NOT EXISTS trained_chara (
+	-- id is the DB-internal ID of the Uma.
+	-- It does not come from the game and should not be supplied in DDL.
+	id INTEGER PRIMARY KEY,
+
+	trainer TEXT NOT NULL,
+	trainer_id INTEGER NOT NULL, -- 0 for NPC umas e.g. in practice races
+	trained_chara_id INTEGER NOT NULL,
+	chara_id INTEGER NOT NULL,
+	chara_name TEXT NOT NULL,
+	-- card_id is the Uma herself, e.g. 100201 for [Innocent Silence] Silence Suzuka.
+	card_id INTEGER NOT NULL,
+	rarity INTEGER NOT NULL,
+	mob_id INTEGER NOT NULL, -- 0 for Umas that are not mobs
+
+	-- Raw stats.
+	raw_speed INTEGER NOT NULL,
+	raw_stamina INTEGER NOT NULL,
+	raw_power INTEGER NOT NULL,
+	raw_guts INTEGER NOT NULL,
+	raw_wit INTEGER NOT NULL,
+
+	-- Aptitudes.
+	turf INTEGER NOT NULL,
+	dirt INTEGER NOT NULL,
+	sprint INTEGER NOT NULL,
+	mile INTEGER NOT NULL,
+	medium INTEGER NOT NULL,
+	long INTEGER NOT NULL,
+	front_runner INTEGER NOT NULL,
+	pace_chaser INTEGER NOT NULL,
+	late_surger INTEGER NOT NULL,
+	end_closer INTEGER NOT NULL,
+
+	-- Other data.
+	fans INTEGER NOT NULL,
+	rank INTEGER NOT NULL,
+	rank_score INTEGER NOT NULL,
+
+	UNIQUE (trainer_id, trained_chara_id)
+) STRICT;
+
+-- Insert trained chara ID 0 to represent no Uma.
+INSERT OR IGNORE INTO trained_chara (
+	id,
+	trainer,
+	trainer_id,
+	trained_chara_id,
+	chara_id, chara_name,
+	card_id,
+	mob_id,
+	raw_speed, raw_stamina, raw_power, raw_guts, raw_wit,
+	turf, dirt,
+	sprint, mile, medium, long,
+	front_runner, pace_chaser, late_surger, end_closer,
+	fans,
+	rank, rank_score
+)
+VALUES (0, '', 0, 0, 0, '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+
+-- trained_chara_skill lists the skills each observed horse owns.
+CREATE TABLE IF NOT EXISTS trained_chara_skill (
+	trained_chara INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	skill_id INTEGER NOT NULL,
+	level INTEGER NOT NULL,
+
+	UNIQUE (trained_chara, skill_id)
+) STRICT;
+CREATE INDEX IF NOT EXISTS trained_chara_skill_by_trained_chara ON trained_chara_skill (trained_chara);
+
+-- race_course_set is the accumulated information of observed race courses.
+CREATE TABLE IF NOT EXISTS race_course_set (
+	id INTEGER PRIMARY KEY,
+	race_track_id INTEGER NOT NULL,
+	distance INTEGER NOT NULL,
+	ground INTEGER NOT NULL,
+	inout INTEGER NOT NULL,
+	turn INTEGER NOT NULL,
+	fence_set INTEGER NOT NULL,
+	float_lane_max INTEGER NOT NULL,
+	course_set_status_id INTEGER NOT NULL,
+	finish_time_min INTEGER NOT NULL,
+	finish_time_min_random_range INTEGER NOT NULL,
+	finish_time_max INTEGER NOT NULL,
+	finish_time_max_random_range INTEGER NOT NULL,
+
+	section_distance REAL NOT NULL,
+	distance_type INTEGER NOT NULL,
+	furlong_number INTEGER NOT NULL,
+	base_speed REAL NOT NULL
+) STRICT;
+
+-- race_config is observed configurations of races including the race type.
+CREATE TABLE IF NOT EXISTS race_config (
+	-- id is the DB-internal ID of the race config.
+	id INTEGER PRIMARY KEY,
+	type INTEGER NOT NULL,
+	course_set_id INTEGER NOT NULL REFERENCES race_course_set (id) ON DELETE RESTRICT,
+	ground_condition INTEGER NOT NULL,
+	weather INTEGER NOT NULL,
+	season INTEGER NOT NULL,
+	time INTEGER NOT NULL,
+	num_horse INTEGER NOT NULL,
+
+	UNIQUE (type, course_set_id, ground_condition, weather, season, time, num_horse)
+) STRICT;
+CREATE INDEX IF NOT EXISTS race_config_by_type ON race_config (type);
+
+-- race is all observed races.
+CREATE TABLE IF NOT EXISTS race (
+	-- id is the DB-internal ID of the race.
+	id INTEGER PRIMARY KEY,
+	config_id INTEGER NOT NULL REFERENCES race_config (id),
+	seed INTEGER NOT NULL,
+
+	distance_diff_max REAL NOT NULL,
+	frame_count INTEGER NOT NULL,
+
+	-- There is no consistent ID for races, and even race seeds have been found
+	-- to collide well above the expected rate for a uniform distribution.
+	-- In order to prevent adding duplicates of races each time the tool is run,
+	-- we include the list of all horse IDs present in the race.
+	-- Analysis on the horses in the race should instead use the race_horse table.
+	-- Horse entries here are 0 where there is actually no horse at that gate.
+	horse_1 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_2 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_3 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_4 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_5 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_6 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_7 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_8 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_9 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_10 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_11 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_12 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_13 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_14 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_15 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_16 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_17 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_18 INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+
+	UNIQUE (
+		config_id,
+		seed,
+		horse_1,
+		horse_2,
+		horse_3,
+		horse_4,
+		horse_5,
+		horse_6,
+		horse_7,
+		horse_8,
+		horse_9,
+		horse_10,
+		horse_11,
+		horse_12,
+		horse_13,
+		horse_14,
+		horse_15,
+		horse_16,
+		horse_17,
+		horse_18
+	)
+) STRICT;
+CREATE INDEX IF NOT EXISTS race_by_config ON race (config_id);
+
+-- race_horse pairs each instance of a horse with its condition and performance
+-- in a given race.
+CREATE TABLE IF NOT EXISTS race_horse (
+	race INTEGER NOT NULL REFERENCES race (id) ON DELETE CASCADE,
+	horse INTEGER NOT NULL REFERENCES trained_chara (id) ON DELETE CASCADE,
+	horse_index INTEGER NOT NULL,
+	post INTEGER NOT NULL,
+	finish_order INTEGER NOT NULL,
+	finish_time_raw REAL NOT NULL,
+	finish_time_scaled REAL NOT NULL,
+	finish_diff_time_from_prev REAL NOT NULL,
+	running_style INTEGER NOT NULL,
+	running_style_ex INTEGER NOT NULL,
+	active_distance_aptitude INTEGER NOT NULL,
+	active_surface_aptitude INTEGER NOT NULL,
+	mood INTEGER NOT NULL,
+	mood_coef REAL NOT NULL,
+	base_speed REAL NOT NULL,
+	base_stamina REAL NOT NULL,
+	base_power REAL NOT NULL,
+	base_guts REAL NOT NULL,
+	base_wit REAL NOT NULL,
+	popularity INTEGER NOT NULL,
+	popularity_rank_left INTEGER NOT NULL,
+	popularity_rank_center INTEGER NOT NULL,
+	popularity_rank_right INTEGER NOT NULL,
+	start_delay REAL NOT NULL,
+	spurt_start_distance REAL NOT NULL,
+
+	PRIMARY KEY (race, horse),
+	UNIQUE (race, horse_index),
+	CHECK (horse != 0)
+) STRICT;
+
+-- race_frame holds the actual received frames.
+CREATE TABLE IF NOT EXISTS race_frame (
+	race INTEGER NOT NULL REFERENCES race (id) ON DELETE CASCADE,
+	frame_time REAL NOT NULL,
+	horse_index INTEGER NOT NULL,
+	distance REAL NOT NULL,
+	lane INTEGER NOT NULL, -- scale -4
+	speed INTEGER NOT NULL, -- scale -2
+	hp INTEGER NOT NULL,
+	temptation INTEGER NOT NULL,
+	block_front_horse_index INTEGER NOT NULL,
+
+	PRIMARY KEY (race, frame_time, horse_index)
+) STRICT;
+
+-- race_event holds raw received race events.
+-- Views such as race_event_skill_activate, race_event_spot_struggle_activate, &c.
+-- interpret events per type.
+CREATE TABLE IF NOT EXISTS race_event (
+	race INTEGER NOT NULL REFERENCES race (id) ON DELETE CASCADE,
+	frame_time REAL NOT NULL,
+	type INTEGER NOT NULL,
+	param_0 INTEGER,
+	param_1 INTEGER,
+	param_2 INTEGER,
+	param_3 INTEGER,
+	param_4 INTEGER,
+	param_5 INTEGER
+) STRICT;
+CREATE INDEX IF NOT EXISTS race_event_by_race ON race_event (race);
+CREATE INDEX IF NOT EXISTS race_event_by_type ON race_event (type);

cmd/racedb/schema-final.sql

@@ -0,0 +1,36 @@
+DROP VIEW IF EXISTS race_event_skill_activate;
+DROP VIEW IF EXISTS race_event_spot_struggle_activate;
+DROP VIEW IF EXISTS race_event_duel_activate;
+DROP VIEW IF EXISTS race_event_release_conserve_power;
+
+CREATE VIEW race_event_skill_activate AS
+SELECT
+	race,
+	frame_time,
+	param_0 AS user_index,
+	param_1 AS skill,
+	param_2 * 0.0001 AS duration,
+	param_3 AS branch,
+	param_4 AS hit_bitmask
+FROM race_event WHERE type = 3;
+
+CREATE VIEW race_event_spot_struggle_activate AS
+SELECT
+	race,
+	frame_time,
+	param_0 AS user_index
+FROM race_event WHERE type = 4;
+
+CREATE VIEW race_event_duel_activate AS
+SELECT
+	race,
+	frame_time,
+	param_0 AS user_index
+FROM race_event WHERE type = 5;
+
+CREATE VIEW race_event_release_conserve_power AS
+SELECT
+	race,
+	frame_time,
+	param_0 AS user_index
+FROM race_event WHERE type = 6;