Wire Format

┌─────────────────────────────────────────────────────────┐
│ Header (4+ bytes)                                       │
├─────────────────────────────────────────────────────────┤
│ Magic: "SJ" (0x53 0x4A)                                 │
│ Version: 0x02                                           │
│ Flags: 0xXX                                             │
├─────────────────────────────────────────────────────────┤
│ [Column Hints] (optional, if FlagHasColumnHints set)   │
├─────────────────────────────────────────────────────────┤
│ Dictionary                                              │
├─────────────────────────────────────────────────────────┤
│ DictLen: varint                                         │
│ Keys: DictLen × [len:varint][UTF-8 bytes]              │
├─────────────────────────────────────────────────────────┤
│ Root Value (recursive encoding)                         │
└─────────────────────────────────────────────────────────┘

53 4A  // "SJ" magic header

0x00  // No compression, no hints
0x01  // Compressed (type in bits 1-2)
0x03  // Compressed with gzip (0x01 | (1 << 1))
0x05  // Compressed with zstd (0x01 | (2 << 1))
0x08  // Column hints present
0x09  // Column hints + compressed

fn encode_uvarint(n: u64) -> Vec<u8> {
    let mut buf = Vec::new();
    let mut val = n;
    loop {
        let mut byte = (val & 0x7F) as u8;
        val >>= 7;
        if val != 0 {
            byte |= 0x80;  // Set continuation bit
        }
        buf.push(byte);
        if val == 0 {
            break;
        }
    }
    buf
}

fn decode_uvarint(data: &[u8]) -> Result<(u64, usize), Error> {
    let mut result = 0u64;
    let mut shift = 0;
    for (i, &byte) in data.iter().enumerate() {
        result |= ((byte & 0x7F) as u64) << shift;
        if (byte & 0x80) == 0 {
            return Ok((result, i + 1));  // Return (value, bytes_read)
        }
        shift += 7;
        if shift >= 64 {
            return Err(Error::VarIntOverflow);
        }
    }
    Err(Error::UnexpectedEof)
}

fn zigzag_encode(n: i64) -> u64 {
    ((n << 1) ^ (n >> 63)) as u64
}

fn zigzag_decode(z: u64) -> i64 {
    ((z >> 1) ^ (-(z & 1) as u64)) as i64
}

00  // Tag only

01  // False
02  // True

Value: 42
Zigzag: 84 (42 << 1)
Varint: 0x54

Wire format: 03 54

Value: -1
Zigzag: 1 ((-1 << 1) ^ (-1 >> 63))
Varint: 0x01

Wire format: 03 01

Value: 3.14159
IEEE 754: 0x400921FB54442D18 (little-endian)

Wire format: 04 18 2D 44 54 FB 21 09 40

Value: "hello"
Length: 5
UTF-8: 68 65 6C 6C 6F

Wire format: 05 05 68 65 6C 6C 6F
             │  │  └──────┬──────┘
             │  │      UTF-8 bytes
             │  └─ Length (varint)
             └─ Tag

[1, 2, 3]

06        // Array tag
03        // Count = 3
03 02     // Int64: 1 (zigzag: 2)
03 04     // Int64: 2 (zigzag: 4)
03 06     // Int64: 3 (zigzag: 6)

{"name": "Alice", "age": 30}

07              // Object tag
02              // Field count = 2
00              // Dict index 0 ("name")
05 05 41 6C 69 63 65  // String: "Alice"
01              // Dict index 1 ("age")
03 3C           // Int64: 30 (zigzag: 60)

┌─────────────────────────────────────────┐
│ Header (4 bytes)                        │
│  - Magic: "SJ"                          │
│  - Version: 0x02                        │
│  - Flags: 0x03 (compressed + gzip)      │
├─────────────────────────────────────────┤
│ OrigLen: varint                         │
│  (uncompressed payload size)            │
├─────────────────────────────────────────┤
│ Compressed Payload                      │
│  (dictionary + root value, compressed)  │
└─────────────────────────────────────────┘

fn encode_compressed(value: &Value, compression: Compression) -> Vec<u8> {
    // 1. Encode uncompressed payload
    let mut payload = Vec::new();
    encode_dictionary(&mut payload);
    encode_value(&mut payload, value);
    
    // 2. Compress payload
    let compressed = match compression {
        Compression::Gzip => gzip_compress(&payload),
        Compression::Zstd => zstd_compress(&payload),
        _ => panic!("Invalid compression type"),
    };
    
    // 3. Build result
    let mut result = Vec::new();
    result.extend_from_slice(b"SJ");  // Magic
    result.push(VERSION);              // Version
    result.push(compression_flags(compression));  // Flags
    write_uvarint(&mut result, payload.len() as u64);  // Original length
    result.extend_from_slice(&compressed);             // Compressed data
    result
}

HintCount: varint
Repeat HintCount times:
  Field: [len:varint][UTF-8 bytes]
  Type: u8
  ShapeLen: varint
  ShapeDims: ShapeLen × varint
  Flags: u8

// Hint: field "embeddings" is a float32 tensor with shape [100, 768]
Field: "embeddings" (10 bytes)
Type: 0x01 (float32)
ShapeLen: 2
ShapeDims: [100, 768]
Flags: 0x00

fn validate_header(data: &[u8]) -> Result<(), Error> {
    if data.len() < 4 {
        return Err(Error::Truncated);
    }
    if data[0] != 0x53 || data[1] != 0x4A {
        return Err(Error::InvalidMagic);
    }
    if data[2] != 0x02 {
        return Err(Error::InvalidVersion(data[2]));
    }
    Ok(())
}