umadump/ism: add missing fields

cmd/ismdb/db.go

@@ -11,10 +11,11 @@ import (
 	"zombiezen.com/go/sqlite/sqlitemigration"
 	"zombiezen.com/go/sqlite/sqlitex"
 
+	"git.sunturtle.xyz/zephyr/horse"
 	"git.sunturtle.xyz/zephyr/horse/umadump/ism"
 )
 
-const appID int32 = 0x4acedb
+const appID int32 = 0x157db
 
 //go:embed schema-00.sql
 var schema0 string
@@ -50,6 +51,8 @@ func opendb(uri, journalMode string) *sqlitemigration.Pool {
 }
 
 func insert(ctx context.Context, db *sqlitemigration.Pool, log *slog.Logger, career *ism.WorkIdleSingleMode, trainerID int64) (err error) {
+	log = log.With(slog.Int64("trainer", trainerID), slog.Int("id", int(career.End.Chara.SingleModeCharaID)))
+
 	conn, err := db.Take(ctx)
 	if err != nil {
 		return err
@@ -57,12 +60,52 @@ func insert(ctx context.Context, db *sqlitemigration.Pool, log *slog.Logger, car
 	defer db.Put(conn)
 
 	defer sqlitex.Transaction(conn)(&err)
+
+	// Do the first insert separately so we can check whether the career is
+	// already in, and so that the join doesn't report the same error repeatedly
+	// if there's something wrong with the db.
+	if err := insertCareer(conn, log, career, trainerID); err != nil {
+		code := sqlite.ErrCode(err)
+		if code == sqlite.ResultConstraintUnique {
+			log.InfoContext(ctx, "already loaded")
+			return nil
+		}
+		return err
+	}
+	trainedChara := career.End.Chara.SingleModeCharaID
+	err = errors.Join(
+		insertConditions(conn, log, career.ProgressLog.CharaEffectLog, trainerID, trainedChara),
+		insertGain(conn, log, &career.ProgressLog.EventGain, trainerID, trainedChara, 0),
+		insertGainSkillHint(conn, log, career.ProgressLog.EventGain.SkillTips, trainerID, trainedChara, 0),
+		insertGain(conn, log, &career.ProgressLog.SuccessionGain, trainerID, trainedChara, 1),
+		insertGainSkillHint(conn, log, career.ProgressLog.SuccessionGain.SkillTips, trainerID, trainedChara, 1),
+	)
+	for _, s := range career.ProgressLog.SupportCardGains {
+		err = errors.Join(
+			err,
+			insertGain(conn, log, &s.GainInfo, trainerID, trainedChara, int32(s.SupportCardID)),
+			insertGainSkillHint(conn, log, s.GainInfo.SkillTips, trainerID, trainedChara, int32(s.SupportCardID)),
+		)
+	}
+	err = errors.Join(
+		err,
+		insertInspirations(conn, log, career.ProgressLog.SuccessionFactorGains, trainerID, trainedChara),
+		insertRaces(conn, log, career.ProgressLog.RaceHistory, trainerID, trainedChara),
+		insertGainedSkills(conn, log, career.ProgressLog.GainSkillIDs, trainerID, trainedChara),
+		insertCareerSkills(conn, log, career.End.Chara.Skills, trainerID, trainedChara),
+		insertCareerSkillHints(conn, log, career.End.Chara.SkillTips, trainerID, trainedChara),
+		insertSupportCards(conn, log, career.End.Chara.SupportCards, trainerID, trainedChara),
+		insertStartSkills(conn, log, career.ProgressInfo.Chara.Skills, trainerID, trainedChara),
+		insertStartSkillHints(conn, log, career.ProgressInfo.Chara.SkillTips, trainerID, trainedChara),
+	)
+	return err
 }
 
 //go:embed insert-career.sql
 var insertCareerSQL string
 
-func insertCareer(conn *sqlite.Conn, career *ism.WorkIdleSingleMode, trainerID int64) error {
+func insertCareer(conn *sqlite.Conn, log *slog.Logger, career *ism.WorkIdleSingleMode, trainerID int64) error {
+	log.Debug("insert career")
 	stmt := conn.Prep(insertCareerSQL)
 	stmt.SetInt64("$trainer_id", trainerID)
 	stmt.SetInt64("$trained_chara_id", int64(career.End.Chara.SingleModeCharaID))
@@ -123,17 +166,18 @@ func insertCareer(conn *sqlite.Conn, career *ism.WorkIdleSingleMode, trainerID i
 	stmt.SetInt64("$reserve_race_program_id", int64(career.End.Chara.ReserveRaceProgramID))
 	stmt.SetInt64("$race_running_style", int64(career.End.Chara.RaceRunningStyle))
 	stmt.SetInt64("$is_short_race", int64(career.End.Chara.IsShortRace))
-	stmt.SetInt64("$succession_trained_chara_id_1", int64(career.OldChara.SuccessionTrainedCharaID1))
+	stmt.SetInt64("$succession_trained_chara_id_1", int64(career.End.Chara.SuccessionTrainedCharaID1))
-	stmt.SetInt64("$succession_trained_chara_id_2", int64(career.OldChara.SuccessionTrainedCharaID2))
+	stmt.SetInt64("$succession_trained_chara_id_2", int64(career.End.Chara.SuccessionTrainedCharaID2))
 	_, err := stmt.Step()
 	err = errors.Join(err, stmt.Reset())
 	if err != nil {
-		return fmt.Errorf("inserting race course: %w", err)
+		return fmt.Errorf("inserting career: %w", err)
 	}
 	return nil
 }
 
-func insertConditions(conn *sqlite.Conn, cs []ism.CharaEffectLog, trainerID int64, trainedChara int32) error {
+func insertConditions(conn *sqlite.Conn, log *slog.Logger, cs []ism.CharaEffectLog, trainerID int64, trainedChara int32) error {
+	log.Debug("insert conditions")
 	const sql = `INSERT INTO condition VALUES (?, ?, ?, ?)`
 	stmt := conn.Prep(sql)
 	stmt.BindInt64(1, trainerID)
@@ -153,7 +197,8 @@ func insertConditions(conn *sqlite.Conn, cs []ism.CharaEffectLog, trainerID int6
 //go:embed insert-gain.sql
 var insertGainSQL string
 
-func insertGain(conn *sqlite.Conn, gain *ism.IdleSingleModeGainInfo, trainerID int64, trainedChara int32, source int32) error {
+func insertGain(conn *sqlite.Conn, log *slog.Logger, gain *ism.IdleSingleModeGainInfo, trainerID int64, trainedChara int32, source int32) error {
+	log.Debug("insert gain", slog.Int("source", int(source)))
 	stmt := conn.Prep(insertGainSQL)
 	stmt.SetInt64("$trainer_id", trainerID)
 	stmt.SetInt64("$trained_chara_id", int64(trainedChara))
@@ -182,7 +227,229 @@ func insertGain(conn *sqlite.Conn, gain *ism.IdleSingleModeGainInfo, trainerID i
 	_, err := stmt.Step()
 	err = errors.Join(err, stmt.Reset())
 	if err != nil {
-		return fmt.Errorf("inserting gain info: %w", err)
+		return fmt.Errorf("inserting gain info for source %d: %w", source, err)
+	}
+	return nil
+}
+
+func insertGainSkillHint(conn *sqlite.Conn, log *slog.Logger, tips []ism.SkillTip, trainerID int64, trainedChara int32, source int32) error {
+	log.Debug("insert gain skill hints", slog.Int("source", int(source)))
+	stmt := conn.Prep(`INSERT INTO gain_skill_hint VALUES (?, ?, ?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	stmt.BindInt64(3, int64(source))
+	for _, t := range tips {
+		stmt.BindInt64(4, int64(t.GroupID))
+		stmt.BindInt64(5, int64(t.Rarity))
+		stmt.BindInt64(6, int64(t.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting skill hint for group %d, rarity %d from source %d: %w", t.GroupID, t.Rarity, source, err)
+		}
+	}
+	return nil
+}
+
+func insertInspirations(conn *sqlite.Conn, log *slog.Logger, inspos []ism.IdleSingleModeSuccessionFactorGain, trainerID int64, trainedChara int32) error {
+	log.Debug("insert inspirations")
+	stmt := conn.Prep(`INSERT INTO inspiration VALUES (?, ?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, yr := range inspos {
+		stmt.BindInt64(3, int64(yr.Year))
+		for _, f := range yr.GainFactorInfo {
+			stmt.BindInt64(4, int64(f.FactorID))
+			stmt.BindInt64(5, int64(f.Level))
+			_, err := stmt.Step()
+			err = errors.Join(err, stmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting spark: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func insertRaces(conn *sqlite.Conn, log *slog.Logger, races []ism.IdleSingleModeRaceHistory, trainerID int64, trainedChara int32) error {
+	log.Debug("insert races")
+	raceStmt := conn.Prep(`INSERT INTO race_history VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	rewardStmt := conn.Prep(`INSERT INTO race_reward VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	raceStmt.BindInt64(1, trainerID)
+	raceStmt.BindInt64(2, int64(trainedChara))
+	rewardStmt.BindInt64(1, trainerID)
+	rewardStmt.BindInt64(2, int64(trainedChara))
+	for _, r := range races {
+		raceStmt.BindInt64(3, int64(r.RaceHistory.Turn))
+		raceStmt.BindInt64(4, int64(r.RaceHistory.ProgramID))
+		raceStmt.BindInt64(5, int64(r.RaceHistory.Weather))
+		raceStmt.BindInt64(6, int64(r.RaceHistory.GroundCondition))
+		raceStmt.BindInt64(7, int64(r.RaceHistory.RunningStyle))
+		raceStmt.BindInt64(8, int64(r.RaceHistory.ResultRank))
+		raceStmt.BindInt64(9, int64(r.RaceHistory.FrameOrder))
+		raceStmt.BindInt64(10, int64(r.RaceRewardInfo.GainedFans))
+		raceStmt.BindInt64(11, int64(r.LoseTipsID))
+		_, err := raceStmt.Step()
+		err = errors.Join(err, raceStmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting race history: %w", err)
+		}
+
+		rewardStmt.BindInt64(3, int64(r.RaceHistory.Turn))
+		rewardStmt.BindBool(7, false)
+		rewardStmt.BindBool(8, false)
+		rewardStmt.BindBool(9, false)
+		for _, reward := range r.RaceRewardInfo.RaceReward {
+			rewardStmt.BindInt64(4, int64(reward.ItemType))
+			rewardStmt.BindInt64(5, int64(reward.ItemID))
+			rewardStmt.BindInt64(6, int64(reward.ItemNum))
+			_, err := rewardStmt.Step()
+			err = errors.Join(err, rewardStmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting race reward: %w", err)
+			}
+		}
+		rewardStmt.BindBool(7, true)
+		for _, reward := range r.RaceRewardInfo.RaceRewardBonus {
+			rewardStmt.BindInt64(4, int64(reward.ItemType))
+			rewardStmt.BindInt64(5, int64(reward.ItemID))
+			rewardStmt.BindInt64(6, int64(reward.ItemNum))
+			_, err := rewardStmt.Step()
+			err = errors.Join(err, rewardStmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting race reward: %w", err)
+			}
+		}
+		rewardStmt.BindBool(7, false)
+		rewardStmt.BindBool(8, true)
+		for _, reward := range r.RaceRewardInfo.RaceRewardPlusBonus {
+			rewardStmt.BindInt64(4, int64(reward.ItemType))
+			rewardStmt.BindInt64(5, int64(reward.ItemID))
+			rewardStmt.BindInt64(6, int64(reward.ItemNum))
+			_, err := rewardStmt.Step()
+			err = errors.Join(err, rewardStmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting race reward: %w", err)
+			}
+		}
+		rewardStmt.BindBool(8, false)
+		rewardStmt.BindBool(9, true)
+		for _, reward := range r.RaceRewardInfo.RaceRewardBonusWin {
+			rewardStmt.BindInt64(4, int64(reward.ItemType))
+			rewardStmt.BindInt64(5, int64(reward.ItemID))
+			rewardStmt.BindInt64(6, int64(reward.ItemNum))
+			_, err := rewardStmt.Step()
+			err = errors.Join(err, rewardStmt.Reset())
+			if err != nil {
+				return fmt.Errorf("inserting race reward: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+func insertGainedSkills(conn *sqlite.Conn, log *slog.Logger, skills []horse.SkillID, trainerID int64, trainedChara int32) error {
+	log.Debug("insert gained skills")
+	stmt := conn.Prep(`INSERT INTO gained_skill VALUES (?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, sk := range skills {
+		stmt.BindInt64(3, int64(sk))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting gain skill: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertCareerSkills(conn *sqlite.Conn, log *slog.Logger, skills []ism.Skill, trainerID int64, trainedChara int32) error {
+	log.Debug("insert career skills")
+	stmt := conn.Prep(`INSERT INTO career_skill VALUES (?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, sk := range skills {
+		stmt.BindInt64(3, int64(sk.ID))
+		stmt.BindInt64(4, int64(sk.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting skill: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertCareerSkillHints(conn *sqlite.Conn, log *slog.Logger, hints []ism.SkillTip, trainerID int64, trainedChara int32) error {
+	log.Debug("insert career skill hints")
+	stmt := conn.Prep(`INSERT INTO career_skill_hint VALUES (?, ?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, h := range hints {
+		stmt.BindInt64(3, int64(h.GroupID))
+		stmt.BindInt64(4, int64(h.Rarity))
+		stmt.BindInt64(5, int64(h.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting skill hint: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertSupportCards(conn *sqlite.Conn, log *slog.Logger, supps []ism.SingleModeSupportCard, trainerID int64, trainedChara int32) error {
+	log.Debug("insert support cards")
+	stmt := conn.Prep(`INSERT INTO career_support_card VALUES (?, ?, ?, ?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, s := range supps {
+		stmt.BindInt64(3, int64(s.Position))
+		stmt.BindInt64(4, int64(s.ID))
+		stmt.BindInt64(5, int64(s.LimitBreak))
+		stmt.BindInt64(6, int64(s.Exp))
+		stmt.BindInt64(7, int64(s.OwnerID))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting support card: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertStartSkills(conn *sqlite.Conn, log *slog.Logger, skills []ism.Skill, trainerID int64, trainedChara int32) error {
+	log.Debug("insert starting skills")
+	stmt := conn.Prep(`INSERT INTO start_skill VALUES (?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, sk := range skills {
+		stmt.BindInt64(3, int64(sk.ID))
+		stmt.BindInt64(4, int64(sk.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting skill: %w", err)
+		}
+	}
+	return nil
+}
+
+func insertStartSkillHints(conn *sqlite.Conn, log *slog.Logger, hints []ism.SkillTip, trainerID int64, trainedChara int32) error {
+	log.Debug("insert starting skill hints")
+	stmt := conn.Prep(`INSERT INTO start_skill_hint VALUES (?, ?, ?, ?, ?)`)
+	stmt.BindInt64(1, trainerID)
+	stmt.BindInt64(2, int64(trainedChara))
+	for _, h := range hints {
+		stmt.BindInt64(3, int64(h.GroupID))
+		stmt.BindInt64(4, int64(h.Rarity))
+		stmt.BindInt64(5, int64(h.Level))
+		_, err := stmt.Step()
+		err = errors.Join(err, stmt.Reset())
+		if err != nil {
+			return fmt.Errorf("inserting skill hint: %w", err)
+		}
 	}
 	return nil
 }

cmd/ismdb/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"log/slog"
+	"os"
+	"os/signal"
+	"path/filepath"
+
+	"git.sunturtle.xyz/zephyr/horse/umadump/ism"
+)
+
+func main() {
+	var (
+		uri, journalMode string
+		fs               []string
+		trainer          int64
+		logLevel         slog.Level
+	)
+	flag.StringVar(&uri, "db", "idle_single_mode.sqlite3", "output database `URI` or filename")
+	flag.StringVar(&journalMode, "journal", "memory", "journal `mode` to apply to database connections")
+	flag.Int64Var(&trainer, "t", 0, "trainer `ID` to associate with the data")
+	flag.TextVar(&logLevel, "log", slog.LevelInfo, "log `level`")
+	flag.Func("d", "`dir`ectory containing idle single mode dumps to load (repeatable)", func(s string) error {
+		es, err := os.ReadDir(s)
+		for _, e := range es {
+			if e.IsDir() {
+				continue
+			}
+			fs = append(fs, filepath.Join(s, e.Name()))
+		}
+		return err
+	})
+	flag.Parse()
+	fs = append(fs, flag.Args()...)
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	go func() {
+		<-ctx.Done()
+		stop()
+	}()
+
+	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
+	db := opendb(uri, journalMode)
+	ok := 0
+	for _, n := range fs {
+		log := log.With(slog.String("file", n))
+		log.Info("begin")
+		f, err := os.Open(n)
+		if err != nil {
+			log.Error("open", slog.Any("err", err))
+			continue
+		}
+		var career ism.WorkIdleSingleMode
+		d := json.NewDecoder(f)
+		d.DisallowUnknownFields()
+		log.Debug("decode")
+		if err := d.Decode(&career); err != nil {
+			log.Error("load", slog.Any("err", err))
+			continue
+		}
+		log.Debug("insert")
+		err = insert(ctx, db, log, &career, trainer)
+		if err != nil {
+			log.Error("insert", slog.Any("err", err))
+			continue
+		}
+		ok++
+	}
+	slog.Info("done", slog.Int("careers", ok))
+}

cmd/ismdb/schema-00.sql

@@ -181,14 +181,13 @@ CREATE TABLE IF NOT EXISTS race_reward (
 CREATE INDEX IF NOT EXISTS race_reward_by_career_turn ON race_reward (trainer_id, trained_chara_id, turn);
 CREATE INDEX IF NOT EXISTS race_reward_by_item ON race_reward (item_id);
 
-CREATE TABLE IF NOT EXISTS gain_skill (
+CREATE TABLE IF NOT EXISTS gained_skill (
 	trainer_id INTEGER NOT NULL DEFAULT 0,
 	trained_chara_id INTEGER NOT NULL,
-	source INTEGER NOT NULL,
 	skill_id INTEGER NOT NULL,
 
-	PRIMARY KEY (trainer_id, trained_chara_id, source, skill_id),
+	PRIMARY KEY (trainer_id, trained_chara_id, skill_id),
-	FOREIGN KEY (trainer_id, trained_chara_id, source) REFERENCES gain ON DELETE CASCADE
+	FOREIGN KEY (trainer_id, trained_chara_id) REFERENCES career ON DELETE CASCADE
 ) STRICT;
 
 CREATE TABLE IF NOT EXISTS career_skill (

umadump/ism/ism.go

@@ -21,7 +21,7 @@ type SingleModeSupportCard struct {
 	ID         horse.SupportCardID `json:"support_card_id"`
 	LimitBreak int32               `json:"limit_break_count"`
 	Exp        int32               `json:"exp"`
-	OwnerID    int32               `json:"owner_viewer_id"`
+	OwnerID    int64               `json:"owner_viewer_id"`
 }
 
 type GroupOuting struct {
@@ -318,6 +318,7 @@ type End struct {
 	TrophyRewardInfo       *umadump.Never  `json:"trophy_reward_info"`
 	PrevCharaGrade         *umadump.Never  `json:"prev_chara_grade"`
 	RaceAddRewardInfo      []umadump.Never `json:"race_add_reward_info"`
+	ReservedRaceInfo       *umadump.Never  `json:"reserved_race_info"`
 	Missions               []umadump.Never `json:"mission_list"`
 	StoryEventMissions     []umadump.Never `json:"story_event_mission_list"`
 	StoryEventCharaBonuses []umadump.Never `json:"story_event_chara_bonus_list"`
@@ -331,10 +332,12 @@ type End struct {
 }
 
 type ProgressInfo struct {
-	Chara     SingleModeChara `json:"chara_info"`
+	Chara                  SingleModeChara `json:"chara_info"`
-	StartTime string          `json:"start_time"`
+	StartTime              string          `json:"start_time"`
-	EndTime   string          `json:"end_time"`
+	EndTime                string          `json:"end_time"`
-	DressID   int32           `json:"dress_id"`
+	DressID                int32           `json:"dress_id"`
+	StoryEventMissions     []umadump.Never `json:"story_event_mission_list"`
+	StoryEventCharaBonuses []umadump.Never `json:"story_event_chara_bonus_list"`
 }
 
 type WorkIdleSingleMode struct {
@@ -342,4 +345,5 @@ type WorkIdleSingleMode struct {
 	ProgressLog  IdleSingleModeProgressLog `json:"progress_log_info"`
 	End          End                       `json:"end_info"`
 	OldChara     *SingleModeChara          `json:"chara_info"`
+	Extra        map[string]any            `json:"extra_data"`
 }