diff --git a/log.go b/log.go new file mode 100644 index 0000000..009a7d5 --- /dev/null +++ b/log.go @@ -0,0 +1,12 @@ +package main + +import ( + "io/ioutil" + "log" + "os" +) + +var ( + Debug = log.New(ioutil.Discard, "DEBUG main: ", 0) + Info = log.New(os.Stdout, "INFO main: ", 0) +) diff --git a/main.go b/main.go index 9e3919f..7bf217e 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "io" + "log" "os" "reflect" "runtime/pprof" @@ -16,6 +17,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/jordanorelli/hyperstone/ent" + "github.com/jordanorelli/hyperstone/stbl" ) const ( @@ -121,6 +123,7 @@ func main() { if opts.v { Debug.SetOutput(os.Stdout) ent.Debug.SetOutput(os.Stdout) + stbl.Debug.SetOutput(os.Stdout) } var handle func(proto.Message) @@ -132,15 +135,13 @@ func main() { case "pretty": handle = prettyPrint case "string-tables": - st := newStringTables() - handle = st.handle + stbl.Debug = log.New(os.Stdout, "", 0) + d := stbl.NewDict() + handle = d.Handle case "class-info": handle = dumpClasses case "entities": handle = dumpEntities - case "baseline": - st := newStringTables() - handle = st.handleBaseline default: bail(1, "no such action: %s", flag.Arg(0)) } diff --git a/stbl/dict.go b/stbl/dict.go new file mode 100644 index 0000000..67648e9 --- /dev/null +++ b/stbl/dict.go @@ -0,0 +1,114 @@ +package stbl + +import ( + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/snappy" + + "github.com/jordanorelli/hyperstone/bit" + "github.com/jordanorelli/hyperstone/dota" +) + +// Dict represents a dictionary of string tables. Each table may be referenced +// by either a numeric ID or its name. +type Dict struct { + byId []Table + byName map[string]*Table + br *bit.BufReader + scratch []byte +} + +func NewDict() *Dict { + return &Dict{ + byId: make([]Table, 0, 64), + byName: make(map[string]*Table, 64), + br: new(bit.BufReader), + scratch: make([]byte, 1<<16), + } +} + +// creates a new table, appending it to our list of tables and adding an entry +// in the table name index +func (d *Dict) newTable(name string) *Table { + d.byId = append(d.byId, Table{name: name}) + t := &d.byId[len(d.byId)-1] + d.byName[name] = t + return t +} + +// Creates a string table based on the provided protobuf message. The table is +// retained in the dict, but a pointer to the table is also returned in case +// the newly-created table is of use to the caller. +func (d *Dict) Create(m *dota.CSVCMsg_CreateStringTable) (*Table, error) { + Debug.Printf("create table %s", m.GetName()) + t := d.newTable(m.GetName()) + + if m.GetUserDataFixedSize() { + t.byteSize = int(m.GetUserDataSize()) + t.bitSize = int(m.GetUserDataSizeBits()) + } + + data := m.GetStringData() + if data == nil || len(data) == 0 { + Debug.Printf("table %s created as empty table", m.GetName()) + return t, nil + } + + if m.GetDataCompressed() { + switch string(data[:4]) { + case "LZSS": + return nil, fmt.Errorf("stbl: LZSS compression is not supported") + default: + var err error + data, err = snappy.Decode(d.scratch, data) + if err != nil { + return nil, fmt.Errorf("stbl: decode error: %v", err) + } + } + } + + d.br.SetSource(data) + if err := t.createEntries(d.br, int(m.GetNumEntries())); err != nil { + return nil, err + } + return t, nil +} + +// updates a string table in the dict based on the data found in a protobuf +// message +func (d *Dict) Update(m *dota.CSVCMsg_UpdateStringTable) error { + Debug.Printf("dict: update %d entries in table having id %d", m.GetNumChangedEntries(), m.GetTableId()) + t := d.TableForId(int(m.GetTableId())) + if t == nil { + return fmt.Errorf("no known string table for id %d", m.GetTableId()) + } + + d.br.SetSource(m.GetStringData()) + return t.updateEntries(d.br, int(m.GetNumChangedEntries())) +} + +func (d *Dict) TableForId(id int) *Table { + if id >= len(d.byId) { + Debug.Printf("bad dict access: id %d is greater than the max table id %d", id, len(d.byId)-1) + return nil + } + return &d.byId[id] +} + +func (d *Dict) TableForName(name string) *Table { + return d.byName[name] +} + +func (d *Dict) Handle(m proto.Message) { + switch v := m.(type) { + case *dota.CSVCMsg_CreateStringTable: + d.Create(v) + case *dota.CSVCMsg_UpdateStringTable: + d.Update(v) + case *dota.CSVCMsg_ClearAllStringTables: + Debug.Println("clear all string tables") + case *dota.CDemoStringTables: + Debug.Println("ignoring a full stringtable dump") + } +} diff --git a/stbl/doc.go b/stbl/doc.go new file mode 100644 index 0000000..7797602 --- /dev/null +++ b/stbl/doc.go @@ -0,0 +1,12 @@ +/* +Package stbl provides facilities for handling String Tables, as defined in +Valve's networking and dem file format. + +String tables are replicated data containers with indexed entries that contain +a text string and optional binary user data (4 kB maximum). String tables are +created on the server and updates are replicated instantly and reliable to all +clients. + +https://developer.valvesoftware.com/wiki/Networking_Events_%26_Messages#String_Tables +*/ +package stbl diff --git a/stbl/entry.go b/stbl/entry.go new file mode 100644 index 0000000..fbcb8a6 --- /dev/null +++ b/stbl/entry.go @@ -0,0 +1,28 @@ +package stbl + +import ( + "fmt" + "unicode/utf8" +) + +// Entry represents a single record in a string table. It's not called "Record" +// because it's called "Entry" in the protobufs. +type Entry struct { + key string + value []byte +} + +func (e Entry) String() string { + if e.value == nil { + return fmt.Sprintf("{%s nil}", e.key) + } + + if utf8.Valid(e.value) { + return fmt.Sprintf("{%s %s}", e.key, e.value) + } + + if len(e.value) > 32 { + return fmt.Sprintf("{%s 0x%x}", e.key, e.value[:32]) + } + return fmt.Sprintf("{%s 0x%x}", e.key, e.value) +} diff --git a/int_ring.go b/stbl/int_ring.go similarity index 98% rename from int_ring.go rename to stbl/int_ring.go index 94d7e83..04d85d7 100644 --- a/int_ring.go +++ b/stbl/int_ring.go @@ -1,4 +1,4 @@ -package main +package stbl type intRing struct { items []int diff --git a/stbl/log.go b/stbl/log.go new file mode 100644 index 0000000..2438e1c --- /dev/null +++ b/stbl/log.go @@ -0,0 +1,12 @@ +package stbl + +import ( + "io/ioutil" + "log" + "os" +) + +var ( + Debug = log.New(ioutil.Discard, "DEBUG stbl: ", 0) + Info = log.New(os.Stdout, "INFO stbl: ", 0) +) diff --git a/stbl/table.go b/stbl/table.go new file mode 100644 index 0000000..cd38f15 --- /dev/null +++ b/stbl/table.go @@ -0,0 +1,122 @@ +package stbl + +import ( + "fmt" + + "github.com/jordanorelli/hyperstone/bit" +) + +type Table struct { + name string + entries []Entry + byteSize int + bitSize int // this is in the protobuf message but I don't know what it does. +} + +// creates n entries from the bit stream br +func (t *Table) createEntries(br *bit.BufReader, n int) error { + Debug.Printf("table %s create %d entries", t.name, n) + t.entries = make([]Entry, n) + var ( + base uint64 + entry *Entry + ) + + for i := range t.entries { + entry = &t.entries[i] + if i > 32 { + base++ + } + + // sequential index flag should always be true in create + if !bit.ReadBool(br) { + return fmt.Errorf("stbl: unexpected nonsequential index") + } + + // key flag: indicates that a key is present + if bit.ReadBool(br) { + // backreading flag: indicates that the key references an earlier + // key or a portion of an earlier key as a prefix + if bit.ReadBool(br) { + entry.key = t.entries[base+br.ReadBits(5)].key[:br.ReadBits(5)] + bit.ReadString(br) + } else { + entry.key = bit.ReadString(br) + } + } + + // value flag: indicates that a value is present + if bit.ReadBool(br) { + if t.byteSize != 0 { + entry.value = make([]byte, t.byteSize) + br.Read(entry.value) + } else { + size := br.ReadBits(14) + br.ReadBits(3) // ??? + entry.value = make([]byte, size) + br.Read(entry.value) + } + } + } + return br.Err() +} + +func (t *Table) updateEntries(br *bit.BufReader, n int) error { + Debug.Printf("table %s update %d entries", t.name, n) + var ( + idx = -1 + entry *Entry + ) + h := newIntRing(32) + for i := 0; i < n; i++ { + // sequential index flag should rarely be true in update + if bit.ReadBool(br) { + idx++ + } else { + idx = int(bit.ReadVarInt(br)) + 1 + } + + h.add(idx) + + // there's probably a faster way to grow the table here. + for idx > len(t.entries)-1 { + t.entries = append(t.entries, Entry{}) + } + + entry = &t.entries[idx] + + // key flag + if bit.ReadBool(br) { + // backreading flag + if bit.ReadBool(br) { + prev, pLen := h.at(int(br.ReadBits(5))), int(br.ReadBits(5)) + if prev < len(t.entries) { + prevEntry := &t.entries[prev] + entry.key = prevEntry.key[:pLen] + bit.ReadString(br) + } else { + return fmt.Errorf("backread error") + } + } else { + entry.key = bit.ReadString(br) + } + } + + // value flag + if bit.ReadBool(br) { + if t.byteSize != 0 { + if entry.value == nil { + entry.value = make([]byte, t.byteSize) + } + } else { + size, _ := int(br.ReadBits(14)), br.ReadBits(3) + if len(entry.value) < size { + entry.value = make([]byte, size) + } else { + entry.value = entry.value[:size] + } + } + br.Read(entry.value) + } + Debug.Printf("%s:%s = %x", t.name, entry.key, entry.value) + } + return nil +} diff --git a/string_tables.go b/string_tables.go deleted file mode 100644 index 3b2a916..0000000 --- a/string_tables.go +++ /dev/null @@ -1,244 +0,0 @@ -package main - -import ( - "fmt" - "os" - "unicode/utf8" - - "github.com/golang/protobuf/proto" - "github.com/golang/snappy" - "github.com/jordanorelli/hyperstone/bit" - "github.com/jordanorelli/hyperstone/dota" -) - -const ( - sTableRingSize = 32 -) - -type stringTables struct { - tables []stringTable - idx map[string]*stringTable - br *bit.BufReader - scratch []byte -} - -func (s *stringTables) String() string { - if s.scratch == nil { - return fmt.Sprintf("{%v %v %v nil}", s.tables, s.idx, s.br) - } - if len(s.scratch) > 32 { - return fmt.Sprintf("{%v %v %v %x...}", s.tables, s.idx, s.br, s.scratch[:32]) - } - return fmt.Sprintf("{%v %v %v %x}", s.tables, s.idx, s.br, s.scratch) -} - -func newStringTables() *stringTables { - return &stringTables{ - tables: make([]stringTable, 0, 64), - idx: make(map[string]*stringTable, 64), - br: new(bit.BufReader), - scratch: make([]byte, 1<<16), - } -} - -type stringTable struct { - name string - entries []stringTableEntry - byteSize int - bitSize int // this is in the protobuf message but I don't know what it does. -} - -func (s stringTable) String() string { - return fmt.Sprintf("{%s %s}", s.name, s.entries) -} - -func (t *stringTable) create(br *bit.BufReader, entries int) { - t.entries = make([]stringTableEntry, entries) - var ( - base uint64 - entry *stringTableEntry - ) - - for i := range t.entries { - entry = &t.entries[i] - if i > 32 { - base++ - } - - // sequential index flag should always be true in create - if !bit.ReadBool(br) { - panic("weird") - } - - // key flag - if bit.ReadBool(br) { - // backreading flag - if bit.ReadBool(br) { - entry.key = t.entries[base+br.ReadBits(5)].key[:br.ReadBits(5)] + bit.ReadString(br) - } else { - entry.key = bit.ReadString(br) - } - } - - // value flag - if bit.ReadBool(br) { - if t.byteSize != 0 { - entry.value = make([]byte, t.byteSize) - br.Read(entry.value) - } else { - size, _ := br.ReadBits(14), br.ReadBits(3) - entry.value = make([]byte, size) - br.Read(entry.value) - } - } - } -} - -type stringTableEntry struct { - key string - value []byte -} - -func (s stringTableEntry) String() string { - if s.value == nil { - return fmt.Sprintf("{%s nil}", s.key) - } - - if utf8.Valid(s.value) { - return fmt.Sprintf("{%s %s}", s.key, s.value) - } - - if len(s.value) > 32 { - return fmt.Sprintf("{%s 0x%x}", s.key, s.value[:32]) - } - return fmt.Sprintf("{%s 0x%x}", s.key, s.value) -} - -func (s *stringTables) handle(m proto.Message) { - switch v := m.(type) { - case *dota.CSVCMsg_CreateStringTable: - s.handleCreate(v) - case *dota.CSVCMsg_UpdateStringTable: - s.handleUpdate(v) - case *dota.CSVCMsg_ClearAllStringTables: - Debug.Println("stringTables: clear all string tables") - case *dota.CDemoStringTables: - Debug.Println("stringTables: ignoring a full stringtable dump") - } -} - -func (s *stringTables) handleBaseline(m proto.Message) { - switch v := m.(type) { - case *dota.CSVCMsg_CreateStringTable: - s.handleCreate(v) - if v.GetName() == "instancebaseline" { - s.idx["instancebaseline"] - fmt.Println("FART FART FART FART FART FART") - } - case *dota.CSVCMsg_UpdateStringTable: - s.handleUpdate(v) - if s.tables[int(v.GetTableId())].name == "instancebaseline" { - fmt.Println("FART FART FART FART FART FART") - } - } -} - -func (s *stringTables) handleCreate(m *dota.CSVCMsg_CreateStringTable) { - Debug.Printf("stringTable create: %s entries: %d", m.GetName(), m.GetNumEntries()) - if m.GetUserDataFixedSize() { - s.tables = append(s.tables, stringTable{name: m.GetName(), byteSize: int(m.GetUserDataSize()), bitSize: int(m.GetUserDataSizeBits())}) - } else { - s.tables = append(s.tables, stringTable{name: m.GetName()}) - } - s.idx[m.GetName()] = &s.tables[len(s.tables)-1] - table := &s.tables[len(s.tables)-1] - - sd := m.GetStringData() - if sd == nil || len(sd) == 0 { - return - } - - if m.GetDataCompressed() { - switch string(sd[:4]) { - case "LZSS": - // TODO: not this - panic("no lzss support!") - default: - var err error - sd, err = snappy.Decode(s.scratch, sd) - if err != nil { - fmt.Fprintf(os.Stderr, "stringtable decode error: %v", err) - return - } - } - } - - s.br.SetSource(sd) - table.create(s.br, int(m.GetNumEntries())) -} - -func (s *stringTables) handleUpdate(m *dota.CSVCMsg_UpdateStringTable) { - // hazard - table := &s.tables[m.GetTableId()] - s.br.SetSource(m.GetStringData()) - table.update(s.br, int(m.GetNumChangedEntries())) -} - -func (t *stringTable) update(br *bit.BufReader, changed int) { - var ( - idx = -1 - entry *stringTableEntry - ) - Debug.Printf("stringTable update table: %s changed: %d", t.name, changed) - h := newIntRing(32) - for i := 0; i < changed; i++ { - // sequential index flag should rarely be true in update - if bit.ReadBool(br) { - idx++ - } else { - idx = int(bit.ReadVarInt(br)) + 1 - } - - h.add(idx) - - for idx > len(t.entries)-1 { - t.entries = append(t.entries, stringTableEntry{}) - } - - entry = &t.entries[idx] - - // key flag - if bit.ReadBool(br) { - // backreading flag - if bit.ReadBool(br) { - prev, pLen := h.at(int(br.ReadBits(5))), int(br.ReadBits(5)) - if prev < len(t.entries) { - prevEntry := &t.entries[prev] - entry.key = prevEntry.key[:pLen] + bit.ReadString(br) - } else { - panic("backread error") - } - } else { - entry.key = bit.ReadString(br) - } - } - - // value flag - if bit.ReadBool(br) { - if t.byteSize != 0 { - if entry.value == nil { - entry.value = make([]byte, t.byteSize) - } - } else { - size, _ := int(br.ReadBits(14)), br.ReadBits(3) - if len(entry.value) < size { - entry.value = make([]byte, size) - } else { - entry.value = entry.value[:size] - } - } - br.Read(entry.value) - } - Debug.Printf("stringTable %s:%s = %x", t.name, entry.key, entry.value) - } -}