jordan
/
hyperstone
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

create stbl package

Browse Source 
to be organized and whatnot
master
Jordan Orelli 8 years ago
parent 96ab319324
commit 64692150ad
9 changed files with 307 additions and 250 deletions
Show Stats Download Patch File Download Diff File

12
log.go
Unescape Escape View File

@ -0,0 +1,12 @@
package main
import (
"io/ioutil"
"log"
"os"
)
var (
Debug = log.New(ioutil.Discard, "DEBUG main: ", 0)
Info = log.New(os.Stdout, "INFO main: ", 0)
)

11
main.go
Unescape Escape View File

@ -8,6 +8,7 @@ import (
"flag"
"fmt"
"io"
"log"
"os"
"reflect"
"runtime/pprof"
@ -16,6 +17,7 @@ import (
"github.com/golang/protobuf/proto"
"github.com/jordanorelli/hyperstone/ent"
"github.com/jordanorelli/hyperstone/stbl"
)
const (
@ -121,6 +123,7 @@ func main() {
if opts.v {
Debug.SetOutput(os.Stdout)
ent.Debug.SetOutput(os.Stdout)
stbl.Debug.SetOutput(os.Stdout)
}
var handle func(proto.Message)
@ -132,15 +135,13 @@ func main() {
case "pretty":
handle = prettyPrint
case "string-tables":
st := newStringTables()
handle = st.handle
stbl.Debug = log.New(os.Stdout, "", 0)
d := stbl.NewDict()
handle = d.Handle
case "class-info":
handle = dumpClasses
case "entities":
handle = dumpEntities
case "baseline":
st := newStringTables()
handle = st.handleBaseline
default:
bail(1, "no such action: %s", flag.Arg(0))
}

114
stbl/dict.go
Unescape Escape View File

@ -0,0 +1,114 @@
package stbl
import (
"fmt"
"github.com/golang/protobuf/proto"
"github.com/golang/snappy"
"github.com/jordanorelli/hyperstone/bit"
"github.com/jordanorelli/hyperstone/dota"
)
// Dict represents a dictionary of string tables. Each table may be referenced
// by either a numeric ID or its name.
type Dict struct {
byId []Table
byName map[string]*Table
br *bit.BufReader
scratch []byte
}
func NewDict() *Dict {
return &Dict{
byId: make([]Table, 0, 64),
byName: make(map[string]*Table, 64),
br: new(bit.BufReader),
scratch: make([]byte, 1<<16),
}
}
// creates a new table, appending it to our list of tables and adding an entry
// in the table name index
func (d *Dict) newTable(name string) *Table {
d.byId = append(d.byId, Table{name: name})
t := &d.byId[len(d.byId)-1]
d.byName[name] = t
return t
}
// Creates a string table based on the provided protobuf message. The table is
// retained in the dict, but a pointer to the table is also returned in case
// the newly-created table is of use to the caller.
func (d *Dict) Create(m *dota.CSVCMsg_CreateStringTable) (*Table, error) {
Debug.Printf("create table %s", m.GetName())
t := d.newTable(m.GetName())
if m.GetUserDataFixedSize() {
t.byteSize = int(m.GetUserDataSize())
t.bitSize = int(m.GetUserDataSizeBits())
}
data := m.GetStringData()
if data == nil || len(data) == 0 {
Debug.Printf("table %s created as empty table", m.GetName())
return t, nil
}
if m.GetDataCompressed() {
switch string(data[:4]) {
case "LZSS":
return nil, fmt.Errorf("stbl: LZSS compression is not supported")
default:
var err error
data, err = snappy.Decode(d.scratch, data)
if err != nil {
return nil, fmt.Errorf("stbl: decode error: %v", err)
}
}
}
d.br.SetSource(data)
if err := t.createEntries(d.br, int(m.GetNumEntries())); err != nil {
return nil, err
}
return t, nil
}
// updates a string table in the dict based on the data found in a protobuf
// message
func (d *Dict) Update(m *dota.CSVCMsg_UpdateStringTable) error {
Debug.Printf("dict: update %d entries in table having id %d", m.GetNumChangedEntries(), m.GetTableId())
t := d.TableForId(int(m.GetTableId()))
if t == nil {
return fmt.Errorf("no known string table for id %d", m.GetTableId())
}
d.br.SetSource(m.GetStringData())
return t.updateEntries(d.br, int(m.GetNumChangedEntries()))
}
func (d *Dict) TableForId(id int) *Table {
if id >= len(d.byId) {
Debug.Printf("bad dict access: id %d is greater than the max table id %d", id, len(d.byId)-1)
return nil
}
return &d.byId[id]
}
func (d *Dict) TableForName(name string) *Table {
return d.byName[name]
}
func (d *Dict) Handle(m proto.Message) {
switch v := m.(type) {
case *dota.CSVCMsg_CreateStringTable:
d.Create(v)
case *dota.CSVCMsg_UpdateStringTable:
d.Update(v)
case *dota.CSVCMsg_ClearAllStringTables:
Debug.Println("clear all string tables")
case *dota.CDemoStringTables:
Debug.Println("ignoring a full stringtable dump")
}
}

12
stbl/doc.go
Unescape Escape View File

@ -0,0 +1,12 @@
/*
Package stbl provides facilities for handling String Tables, as defined in
Valve's networking and dem file format.
String tables are replicated data containers with indexed entries that contain
a text string and optional binary user data (4 kB maximum). String tables are
created on the server and updates are replicated instantly and reliable to all
clients.
https://developer.valvesoftware.com/wiki/Networking_Events_%26_Messages#String_Tables
*/
package stbl

28
stbl/entry.go
Unescape Escape View File

@ -0,0 +1,28 @@
package stbl
import (
"fmt"
"unicode/utf8"
)
// Entry represents a single record in a string table. It's not called "Record"
// because it's called "Entry" in the protobufs.
type Entry struct {
key string
value []byte
}
func (e Entry) String() string {
if e.value == nil {
return fmt.Sprintf("{%s nil}", e.key)
}
if utf8.Valid(e.value) {
return fmt.Sprintf("{%s %s}", e.key, e.value)
}
if len(e.value) > 32 {
return fmt.Sprintf("{%s 0x%x}", e.key, e.value[:32])
}
return fmt.Sprintf("{%s 0x%x}", e.key, e.value)
}

2
int_ring.go → stbl/int_ring.go
Unescape Escape View File

@ -1,4 +1,4 @@
package main
package stbl
type intRing struct {
items []int

12
stbl/log.go
Unescape Escape View File

@ -0,0 +1,12 @@
package stbl
import (
"io/ioutil"
"log"
"os"
)
var (
Debug = log.New(ioutil.Discard, "DEBUG stbl: ", 0)
Info = log.New(os.Stdout, "INFO stbl: ", 0)
)

122
stbl/table.go
Unescape Escape View File

@ -0,0 +1,122 @@
package stbl
import (
"fmt"
"github.com/jordanorelli/hyperstone/bit"
)
type Table struct {
name string
entries []Entry
byteSize int
bitSize int // this is in the protobuf message but I don't know what it does.
}
// creates n entries from the bit stream br
func (t *Table) createEntries(br *bit.BufReader, n int) error {
Debug.Printf("table %s create %d entries", t.name, n)
t.entries = make([]Entry, n)
var (
base uint64
entry *Entry
)
for i := range t.entries {
entry = &t.entries[i]
if i > 32 {
base++
}
// sequential index flag should always be true in create
if !bit.ReadBool(br) {
return fmt.Errorf("stbl: unexpected nonsequential index")
}
// key flag: indicates that a key is present
if bit.ReadBool(br) {
// backreading flag: indicates that the key references an earlier
// key or a portion of an earlier key as a prefix
if bit.ReadBool(br) {
entry.key = t.entries[base+br.ReadBits(5)].key[:br.ReadBits(5)] + bit.ReadString(br)
} else {
entry.key = bit.ReadString(br)
}
}
// value flag: indicates that a value is present
if bit.ReadBool(br) {
if t.byteSize != 0 {
entry.value = make([]byte, t.byteSize)
br.Read(entry.value)
} else {
size := br.ReadBits(14)
br.ReadBits(3) // ???
entry.value = make([]byte, size)
br.Read(entry.value)
}
}
}
return br.Err()
}
func (t *Table) updateEntries(br *bit.BufReader, n int) error {
Debug.Printf("table %s update %d entries", t.name, n)
var (
idx = -1
entry *Entry
)
h := newIntRing(32)
for i := 0; i < n; i++ {
// sequential index flag should rarely be true in update
if bit.ReadBool(br) {
idx++
} else {
idx = int(bit.ReadVarInt(br)) + 1
}
h.add(idx)
// there's probably a faster way to grow the table here.
for idx > len(t.entries)-1 {
t.entries = append(t.entries, Entry{})
}
entry = &t.entries[idx]
// key flag
if bit.ReadBool(br) {
// backreading flag
if bit.ReadBool(br) {
prev, pLen := h.at(int(br.ReadBits(5))), int(br.ReadBits(5))
if prev < len(t.entries) {
prevEntry := &t.entries[prev]
entry.key = prevEntry.key[:pLen] + bit.ReadString(br)
} else {
return fmt.Errorf("backread error")
}
} else {
entry.key = bit.ReadString(br)
}
}
// value flag
if bit.ReadBool(br) {
if t.byteSize != 0 {
if entry.value == nil {
entry.value = make([]byte, t.byteSize)
}
} else {
size, _ := int(br.ReadBits(14)), br.ReadBits(3)
if len(entry.value) < size {
entry.value = make([]byte, size)
} else {
entry.value = entry.value[:size]
}
}
br.Read(entry.value)
}
Debug.Printf("%s:%s = %x", t.name, entry.key, entry.value)
}
return nil
}

244
string_tables.go
Unescape Escape View File

@ -1,244 +0,0 @@
package main
import (
"fmt"
"os"
"unicode/utf8"
"github.com/golang/protobuf/proto"
"github.com/golang/snappy"
"github.com/jordanorelli/hyperstone/bit"
"github.com/jordanorelli/hyperstone/dota"
)
const (
sTableRingSize = 32
)
type stringTables struct {
tables []stringTable
idx map[string]*stringTable
br *bit.BufReader
scratch []byte
}
func (s *stringTables) String() string {
if s.scratch == nil {
return fmt.Sprintf("{%v %v %v nil}", s.tables, s.idx, s.br)
}
if len(s.scratch) > 32 {
return fmt.Sprintf("{%v %v %v %x...}", s.tables, s.idx, s.br, s.scratch[:32])
}
return fmt.Sprintf("{%v %v %v %x}", s.tables, s.idx, s.br, s.scratch)
}
func newStringTables() *stringTables {
return &stringTables{
tables: make([]stringTable, 0, 64),
idx: make(map[string]*stringTable, 64),
br: new(bit.BufReader),
scratch: make([]byte, 1<<16),
}
}
type stringTable struct {
name string
entries []stringTableEntry
byteSize int
bitSize int // this is in the protobuf message but I don't know what it does.
}
func (s stringTable) String() string {
return fmt.Sprintf("{%s %s}", s.name, s.entries)
}
func (t *stringTable) create(br *bit.BufReader, entries int) {
t.entries = make([]stringTableEntry, entries)
var (
base uint64
entry *stringTableEntry
)
for i := range t.entries {
entry = &t.entries[i]
if i > 32 {
base++
}
// sequential index flag should always be true in create
if !bit.ReadBool(br) {
panic("weird")
}
// key flag
if bit.ReadBool(br) {
// backreading flag
if bit.ReadBool(br) {
entry.key = t.entries[base+br.ReadBits(5)].key[:br.ReadBits(5)] + bit.ReadString(br)
} else {
entry.key = bit.ReadString(br)
}
}
// value flag
if bit.ReadBool(br) {
if t.byteSize != 0 {
entry.value = make([]byte, t.byteSize)
br.Read(entry.value)
} else {
size, _ := br.ReadBits(14), br.ReadBits(3)
entry.value = make([]byte, size)
br.Read(entry.value)
}
}
}
}
type stringTableEntry struct {
key string
value []byte
}
func (s stringTableEntry) String() string {
if s.value == nil {
return fmt.Sprintf("{%s nil}", s.key)
}
if utf8.Valid(s.value) {
return fmt.Sprintf("{%s %s}", s.key, s.value)
}
if len(s.value) > 32 {
return fmt.Sprintf("{%s 0x%x}", s.key, s.value[:32])
}
return fmt.Sprintf("{%s 0x%x}", s.key, s.value)
}
func (s *stringTables) handle(m proto.Message) {
switch v := m.(type) {
case *dota.CSVCMsg_CreateStringTable:
s.handleCreate(v)
case *dota.CSVCMsg_UpdateStringTable:
s.handleUpdate(v)
case *dota.CSVCMsg_ClearAllStringTables:
Debug.Println("stringTables: clear all string tables")
case *dota.CDemoStringTables:
Debug.Println("stringTables: ignoring a full stringtable dump")
}
}
func (s *stringTables) handleBaseline(m proto.Message) {
switch v := m.(type) {
case *dota.CSVCMsg_CreateStringTable:
s.handleCreate(v)
if v.GetName() == "instancebaseline" {
s.idx["instancebaseline"]
fmt.Println("FART FART FART FART FART FART")
}
case *dota.CSVCMsg_UpdateStringTable:
s.handleUpdate(v)
if s.tables[int(v.GetTableId())].name == "instancebaseline" {
fmt.Println("FART FART FART FART FART FART")
}
}
}
func (s *stringTables) handleCreate(m *dota.CSVCMsg_CreateStringTable) {
Debug.Printf("stringTable create: %s entries: %d", m.GetName(), m.GetNumEntries())
if m.GetUserDataFixedSize() {
s.tables = append(s.tables, stringTable{name: m.GetName(), byteSize: int(m.GetUserDataSize()), bitSize: int(m.GetUserDataSizeBits())})
} else {
s.tables = append(s.tables, stringTable{name: m.GetName()})
}
s.idx[m.GetName()] = &s.tables[len(s.tables)-1]
table := &s.tables[len(s.tables)-1]
sd := m.GetStringData()
if sd == nil || len(sd) == 0 {
return
}
if m.GetDataCompressed() {
switch string(sd[:4]) {
case "LZSS":
// TODO: not this
panic("no lzss support!")
default:
var err error
sd, err = snappy.Decode(s.scratch, sd)
if err != nil {
fmt.Fprintf(os.Stderr, "stringtable decode error: %v", err)
return
}
}
}
s.br.SetSource(sd)
table.create(s.br, int(m.GetNumEntries()))
}
func (s *stringTables) handleUpdate(m *dota.CSVCMsg_UpdateStringTable) {
// hazard
table := &s.tables[m.GetTableId()]
s.br.SetSource(m.GetStringData())
table.update(s.br, int(m.GetNumChangedEntries()))
}
func (t *stringTable) update(br *bit.BufReader, changed int) {
var (
idx = -1
entry *stringTableEntry
)
Debug.Printf("stringTable update table: %s changed: %d", t.name, changed)
h := newIntRing(32)
for i := 0; i < changed; i++ {
// sequential index flag should rarely be true in update
if bit.ReadBool(br) {
idx++
} else {
idx = int(bit.ReadVarInt(br)) + 1
}
h.add(idx)
for idx > len(t.entries)-1 {
t.entries = append(t.entries, stringTableEntry{})
}
entry = &t.entries[idx]
// key flag
if bit.ReadBool(br) {
// backreading flag
if bit.ReadBool(br) {
prev, pLen := h.at(int(br.ReadBits(5))), int(br.ReadBits(5))
if prev < len(t.entries) {
prevEntry := &t.entries[prev]
entry.key = prevEntry.key[:pLen] + bit.ReadString(br)
} else {
panic("backread error")
}
} else {
entry.key = bit.ReadString(br)
}
}
// value flag
if bit.ReadBool(br) {
if t.byteSize != 0 {
if entry.value == nil {
entry.value = make([]byte, t.byteSize)
}
} else {
size, _ := int(br.ReadBits(14)), br.ReadBits(3)
if len(entry.value) < size {
entry.value = make([]byte, size)
} else {
entry.value = entry.value[:size]
}
}
br.Read(entry.value)
}
Debug.Printf("stringTable %s:%s = %x", t.name, entry.key, entry.value)
}
}
Write Preview
Loading…
Cancel
Save