feat: add StorageAdapter abstraction and InMemoryAdapter with full test suite

- Introduce StorageAdapter protocol and InMemoryAdapter (thread-safe, in-memory)
  in engine/adapter.py to decouple storage from sync logic
- Refactor SyncEngine to accept a StorageAdapter, replacing the previous
  stub that returned an empty list
- Wire server to use SyncEngine + InMemoryAdapter, removing hardcoded
  SYNC_OPERATIONS fixtures
- Add tests for engine, adapter, generator, and types (tests/)
- Add pre-commit config (.pre-commit-config.yaml) with linting/formatting hooks
- Update pyproject.toml and uv.lock with new dev dependencies
- Clean up type annotations and CLI help strings across generator, server, and utils
This commit is contained in:
2026-08-27 16:48:15 +03:30
parent b8449a6ff3
commit 5c8310b837
24 changed files with 1735 additions and 118 deletions
View File
+148
View File
@@ -0,0 +1,148 @@
"""Tests for engine.adapter — InMemoryAdapter and StorageAdapter protocol."""
import threading
from engine.adapter import InMemoryAdapter
from engine.types.operation import Operation
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_op(**kwargs: object) -> Operation:
defaults: dict[str, object] = {
"type": "insert",
"device_id": "dev-1",
"timestamp": "1000000.0",
"entity_id": "e1",
"entity_type": "obj",
"payload": {"key": "value"},
}
defaults.update(kwargs)
return Operation(**defaults)
# ---------------------------------------------------------------------------
# Initial state
# ---------------------------------------------------------------------------
class TestInMemoryAdapterInit:
def test_cursor_starts_at_zero(self) -> None:
adapter = InMemoryAdapter()
assert adapter.cursor == 0
def test_get_all_returns_empty_on_fresh_adapter(self) -> None:
adapter = InMemoryAdapter()
assert adapter.get_operations_after(0) == []
# ---------------------------------------------------------------------------
# append_operations
# ---------------------------------------------------------------------------
class TestAppendOperations:
def test_returns_new_cursor(self) -> None:
adapter = InMemoryAdapter()
cursor = adapter.append_operations([make_op(), make_op()])
assert cursor == 2
def test_empty_append_returns_current_cursor(self) -> None:
adapter = InMemoryAdapter()
adapter.append_operations([make_op()])
cursor = adapter.append_operations([])
assert cursor == 1
def test_sequential_ids_are_stamped(self) -> None:
adapter = InMemoryAdapter()
adapter.append_operations([make_op(), make_op(), make_op()])
ids = [op.id for op in adapter._log]
assert ids == ["1", "2", "3"]
def test_original_op_id_is_overwritten(self) -> None:
"""Server always assigns its own sequence number."""
adapter = InMemoryAdapter()
adapter.append_operations([make_op(id="client-assigned-id")])
assert adapter._log[0].id == "1"
def test_original_op_fields_preserved(self) -> None:
adapter = InMemoryAdapter()
op = make_op(entity_id="entity-42", payload={"hello": "world"})
adapter.append_operations([op])
stored = adapter._log[0]
assert stored.entity_id == "entity-42"
assert stored.payload == {"hello": "world"}
def test_cursor_advances_incrementally(self) -> None:
adapter = InMemoryAdapter()
c1 = adapter.append_operations([make_op()])
c2 = adapter.append_operations([make_op()])
c3 = adapter.append_operations([make_op()])
assert c1 == 1
assert c2 == 2
assert c3 == 3
# ---------------------------------------------------------------------------
# get_operations_after
# ---------------------------------------------------------------------------
class TestGetOperationsAfter:
def test_cursor_zero_returns_all(self) -> None:
adapter = InMemoryAdapter()
adapter.append_operations([make_op(), make_op()])
ops = adapter.get_operations_after(0)
assert len(ops) == 2
def test_cursor_at_end_returns_empty(self) -> None:
adapter = InMemoryAdapter()
adapter.append_operations([make_op(), make_op()])
ops = adapter.get_operations_after(2)
assert ops == []
def test_cursor_in_middle_returns_tail(self) -> None:
adapter = InMemoryAdapter()
adapter.append_operations([make_op(), make_op(), make_op()])
ops = adapter.get_operations_after(1)
assert len(ops) == 2
assert ops[0].id == "2"
assert ops[1].id == "3"
def test_returns_copy_not_reference(self) -> None:
"""Mutating the returned list must not affect the internal log."""
adapter = InMemoryAdapter()
adapter.append_operations([make_op()])
ops = adapter.get_operations_after(0)
ops.clear()
assert len(adapter._log) == 1
# ---------------------------------------------------------------------------
# Thread safety
# ---------------------------------------------------------------------------
class TestThreadSafety:
def test_concurrent_appends_produce_unique_sequential_ids(self) -> None:
adapter = InMemoryAdapter()
errors: list[Exception] = []
def worker() -> None:
try:
adapter.append_operations([make_op()])
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(50)]
for t in threads:
t.start()
for t in threads:
t.join()
assert errors == [], f"Thread errors: {errors}"
assert adapter.cursor == 50
ids = {op.id for op in adapter._log}
assert ids == {str(i) for i in range(1, 51)}
+167
View File
@@ -0,0 +1,167 @@
"""Tests for engine.SyncEngine — cursor semantics and bidirectional sync."""
from engine import SyncEngine
from engine.adapter import InMemoryAdapter
from engine.types.operation import Operation
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_op(entity_id: str = "e1", device_id: str = "dev-1") -> Operation:
return Operation(
type="insert",
device_id=device_id,
timestamp="1000000.0",
entity_id=entity_id,
entity_type="obj",
payload={},
)
# ---------------------------------------------------------------------------
# Initialisation
# ---------------------------------------------------------------------------
class TestSyncEngineInit:
def test_default_adapter_is_in_memory(self) -> None:
engine = SyncEngine()
assert isinstance(engine.adapter, InMemoryAdapter)
def test_custom_adapter_is_stored(self) -> None:
adapter = InMemoryAdapter()
engine = SyncEngine(adapter=adapter)
assert engine.adapter is adapter
def test_none_adapter_falls_back_to_in_memory(self) -> None:
engine = SyncEngine(adapter=None)
assert isinstance(engine.adapter, InMemoryAdapter)
# ---------------------------------------------------------------------------
# Cursor semantics
# ---------------------------------------------------------------------------
class TestCursorSemantics:
def test_fresh_sync_with_no_ops_returns_zero_cursor(self) -> None:
engine = SyncEngine()
cursor, ops = engine.sync(cursor=0, operations=[])
assert cursor == 0
assert ops == []
def test_cursor_advances_by_number_of_new_ops(self) -> None:
engine = SyncEngine()
cursor, ops = engine.sync(cursor=0, operations=[make_op(), make_op()])
assert cursor == 2
assert len(ops) == 2
def test_second_sync_at_new_cursor_returns_nothing_new(self) -> None:
engine = SyncEngine()
cursor, _ = engine.sync(cursor=0, operations=[make_op()])
cursor2, ops2 = engine.sync(cursor=cursor, operations=[])
assert cursor2 == cursor
assert ops2 == []
def test_cursor_is_stable_across_empty_syncs(self) -> None:
engine = SyncEngine()
engine.sync(cursor=0, operations=[make_op(), make_op()])
c1, _ = engine.sync(cursor=2, operations=[])
c2, _ = engine.sync(cursor=2, operations=[])
assert c1 == c2 == 2
# ---------------------------------------------------------------------------
# Bidirectional sync
# ---------------------------------------------------------------------------
class TestBidirectionalSync:
def test_client_ops_are_visible_to_next_client(self) -> None:
"""Op sent by client-1 must appear in client-2's next sync."""
engine = SyncEngine()
# client-1 sends one op
engine.sync(cursor=0, operations=[make_op(device_id="dev-1")])
# client-2 syncs from 0 — should see client-1's op
_, ops = engine.sync(cursor=0, operations=[])
assert len(ops) == 1
assert ops[0].device_id == "dev-1"
def test_client_receives_own_ops_back_on_first_sync(self) -> None:
"""Ops are appended then fetched, so the sender gets them back too."""
engine = SyncEngine()
_cursor, ops = engine.sync(cursor=0, operations=[make_op(), make_op()])
assert len(ops) == 2
def test_client_does_not_receive_already_seen_ops(self) -> None:
engine = SyncEngine()
# client-1 first sync: sends 2 ops, receives them back at cursor=2
cursor, _ = engine.sync(cursor=0, operations=[make_op(), make_op()])
# client-1 second sync: sends nothing, cursor already at 2 → empty
_, ops = engine.sync(cursor=cursor, operations=[])
assert ops == []
def test_two_clients_interleaved(self) -> None:
engine = SyncEngine()
# client-1 syncs first, sends 1 op (cursor 0→1)
c1, _ = engine.sync(cursor=0, operations=[make_op(device_id="dev-1")])
assert c1 == 1
# client-2 syncs from 0, sends 1 op (cursor 0→2, sees both)
c2, ops2 = engine.sync(cursor=0, operations=[make_op(device_id="dev-2")])
assert c2 == 2
assert len(ops2) == 2
# client-1 syncs again from 1, receives only the new op from dev-2
c1b, ops1b = engine.sync(cursor=c1, operations=[])
assert c1b == 2
assert len(ops1b) == 1
assert ops1b[0].device_id == "dev-2"
def test_server_assigned_ids_are_sequential(self) -> None:
engine = SyncEngine()
_, ops = engine.sync(cursor=0, operations=[make_op(), make_op(), make_op()])
assert [op.id for op in ops] == ["1", "2", "3"]
# ---------------------------------------------------------------------------
# Append-only (no conflict resolution)
# ---------------------------------------------------------------------------
class TestAppendOnly:
def test_conflicting_ops_on_same_entity_both_stored(self) -> None:
"""Both ops survive; no merging or rejection."""
engine = SyncEngine()
op1 = Operation(
type="update",
device_id="dev-1",
timestamp="1.0",
entity_id="shared",
entity_type="obj",
payload={"v": 1},
)
op2 = Operation(
type="update",
device_id="dev-2",
timestamp="2.0",
entity_id="shared",
entity_type="obj",
payload={"v": 2},
)
engine.sync(cursor=0, operations=[op1])
engine.sync(cursor=0, operations=[op2])
_, all_ops = engine.sync(cursor=0, operations=[])
assert len(all_ops) == 2
payloads = [op.payload for op in all_ops]
assert {"v": 1} in payloads
assert {"v": 2} in payloads
+314
View File
@@ -0,0 +1,314 @@
"""Tests for generator.languages.python — unit and integration."""
import json
from pathlib import Path
import pytest
from generator.languages.python import PythonGenerator
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
OPERATION_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://example.com/operation.schema.json",
"title": "Operation",
"description": "A single operation.",
"type": "object",
"properties": {
"id": {"type": "string", "description": "Unique id."},
"type": {"type": "string", "description": "Op type."},
"count": {"type": "integer", "description": "Count."},
"score": {"type": "number", "description": "Score."},
"active": {"type": "boolean", "description": "Active flag."},
"meta": {"type": "object", "description": "Metadata."},
},
}
ITEM_SCHEMA = {
"title": "Item",
"description": "An item.",
"type": "object",
"properties": {
"name": {"type": "string", "description": "Item name."},
},
"required": ["name"],
}
CONTAINER_SCHEMA = {
"title": "Container",
"description": "A container of items.",
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {"$ref": "item.schema.json"},
"description": "Items list.",
},
"label": {"type": "string", "description": "Label."},
},
"required": ["label"],
}
@pytest.fixture()
def schema_dir(tmp_path: Path) -> Path:
"""Write a minimal set of schemas to a temp directory and return its path."""
(tmp_path / "operation.schema.json").write_text(json.dumps(OPERATION_SCHEMA))
return tmp_path
@pytest.fixture()
def ref_schema_dir(tmp_path: Path) -> Path:
"""Write item + container schemas (cross-ref) to a temp directory."""
(tmp_path / "item.schema.json").write_text(json.dumps(ITEM_SCHEMA))
(tmp_path / "container.schema.json").write_text(json.dumps(CONTAINER_SCHEMA))
return tmp_path
# ---------------------------------------------------------------------------
# _schema_to_module
# ---------------------------------------------------------------------------
class TestSchemaToModule:
@pytest.fixture(autouse=True)
def gen(self, schema_dir: Path) -> None:
self.gen = PythonGenerator(schema_dir)
def test_removes_schema_json_suffix(self) -> None:
assert self.gen._schema_to_module("operation.schema.json") == "operation"
def test_replaces_hyphens_with_underscores(self) -> None:
assert self.gen._schema_to_module("sync-request.schema.json") == "sync_request"
def test_combined(self) -> None:
assert self.gen._schema_to_module("my-type.schema.json") == "my_type"
# ---------------------------------------------------------------------------
# _ref_to_filename
# ---------------------------------------------------------------------------
class TestRefToFilename:
@pytest.fixture(autouse=True)
def gen(self, schema_dir: Path) -> None:
self.gen = PythonGenerator(schema_dir)
def test_bare_filename(self) -> None:
result = self.gen._ref_to_filename("operation.schema.json")
assert result == "operation.schema.json"
def test_relative_prefix(self) -> None:
result = self.gen._ref_to_filename("./operation.schema.json")
assert result == "operation.schema.json"
def test_nested_path(self) -> None:
result = self.gen._ref_to_filename("schemas/operation.schema.json")
assert result == "operation.schema.json"
# ---------------------------------------------------------------------------
# _find_refs
# ---------------------------------------------------------------------------
class TestFindRefs:
@pytest.fixture(autouse=True)
def gen(self, schema_dir: Path) -> None:
self.gen = PythonGenerator(schema_dir)
def test_no_refs(self) -> None:
schema = {"type": "object", "properties": {"id": {"type": "string"}}}
assert self.gen._find_refs(schema) == []
def test_top_level_ref(self) -> None:
schema = {"$ref": "other.schema.json"}
assert self.gen._find_refs(schema) == ["other.schema.json"]
def test_nested_ref_in_properties(self) -> None:
schema = {
"type": "object",
"properties": {
"child": {"$ref": "child.schema.json"},
},
}
refs = self.gen._find_refs(schema)
assert "child.schema.json" in refs
def test_ref_inside_array_items(self) -> None:
schema = {
"type": "array",
"items": {"$ref": "item.schema.json"},
}
refs = self.gen._find_refs(schema)
assert "item.schema.json" in refs
# ---------------------------------------------------------------------------
# _resolve_type
# ---------------------------------------------------------------------------
class TestResolveType:
@pytest.fixture(autouse=True)
def gen(self, schema_dir: Path) -> None:
self.gen = PythonGenerator(schema_dir)
@pytest.mark.parametrize(
"json_type,expected",
[
("string", "str"),
("integer", "int"),
("number", "float"),
("boolean", "bool"),
("object", "dict"),
],
)
def test_primitive_types(self, json_type: str, expected: str) -> None:
info = self.gen._resolve_type({"type": json_type})
assert info.type == expected
assert info.imports == set()
def test_array_of_string(self) -> None:
info = self.gen._resolve_type({"type": "array", "items": {"type": "string"}})
assert info.type == "list[str]"
def test_format_int64(self) -> None:
info = self.gen._resolve_type({"type": "integer", "format": "int64"})
assert info.type == "int"
def test_unsupported_type_raises(self) -> None:
with pytest.raises(ValueError, match="Unsupported schema"):
self.gen._resolve_type({"type": "unknown_type"})
# ---------------------------------------------------------------------------
# _parse_model
# ---------------------------------------------------------------------------
class TestParseModel:
@pytest.fixture(autouse=True)
def gen(self, schema_dir: Path) -> None:
self.gen = PythonGenerator(schema_dir)
def test_model_name_from_title(self) -> None:
model = self.gen._parse_model(OPERATION_SCHEMA)
assert model.name == "Operation"
def test_model_description(self) -> None:
model = self.gen._parse_model(OPERATION_SCHEMA)
assert model.description == "A single operation."
def test_all_fields_optional_when_no_required(self) -> None:
model = self.gen._parse_model(OPERATION_SCHEMA)
assert all(f.optional for f in model.fields)
def test_required_fields_not_optional(self) -> None:
schema = {
"title": "Req",
"description": "desc",
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"},
},
"required": ["name"],
}
model = self.gen._parse_model(schema)
name_field = next(f for f in model.fields if f.name == "name")
age_field = next(f for f in model.fields if f.name == "age")
assert not name_field.optional
assert age_field.optional
def test_optional_fields_have_none_default_in_type(self) -> None:
model = self.gen._parse_model(OPERATION_SCHEMA)
for field in model.fields:
assert "| None = None" in field.type
def test_required_fields_sorted_before_optional(self) -> None:
schema = {
"title": "T",
"description": "",
"type": "object",
"properties": {
"opt_field": {"type": "string"},
"req_field": {"type": "integer"},
},
"required": ["req_field"],
}
model = self.gen._parse_model(schema)
assert model.fields[0].name == "req_field"
# ---------------------------------------------------------------------------
# Integration — generate single schema
# ---------------------------------------------------------------------------
class TestGenerateIntegration:
def test_generates_python_file(self, schema_dir: Path, tmp_path: Path) -> None:
gen = PythonGenerator(schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=False, schema_file="operation.schema.json")
assert (out / "operation.py").exists()
def test_generated_file_contains_class(
self, schema_dir: Path, tmp_path: Path
) -> None:
gen = PythonGenerator(schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=False, schema_file="operation.schema.json")
content = (out / "operation.py").read_text()
assert "class Operation(BaseModel):" in content
def test_generated_file_has_encode_decode(
self, schema_dir: Path, tmp_path: Path
) -> None:
gen = PythonGenerator(schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=False, schema_file="operation.schema.json")
content = (out / "operation.py").read_text()
assert "def encode" in content
assert "def decode" in content
def test_generate_all_creates_all_files(
self, schema_dir: Path, tmp_path: Path
) -> None:
gen = PythonGenerator(schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=True, schema_file=None)
assert (out / "operation.py").exists()
def test_cross_ref_generates_dependency_first(
self, ref_schema_dir: Path, tmp_path: Path
) -> None:
"""Generating container.schema.json must also produce item.py."""
gen = PythonGenerator(ref_schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=False, schema_file="container.schema.json")
assert (out / "item.py").exists()
assert (out / "container.py").exists()
def test_cross_ref_import_present_in_generated_file(
self, ref_schema_dir: Path, tmp_path: Path
) -> None:
gen = PythonGenerator(ref_schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=False, schema_file="container.schema.json")
content = (out / "container.py").read_text()
assert "from .item import Item" in content
def test_generate_all_is_idempotent(self, schema_dir: Path, tmp_path: Path) -> None:
"""Running generate twice must not raise or produce duplicate content."""
gen = PythonGenerator(schema_dir)
out = tmp_path / "out"
gen.generate(out, make_all=True, schema_file=None)
content_first = (out / "operation.py").read_text()
gen.generate(out, make_all=True, schema_file=None)
content_second = (out / "operation.py").read_text()
assert content_first == content_second
+169
View File
@@ -0,0 +1,169 @@
"""Tests for all Pydantic models — encode/decode roundtrips and field contracts."""
import json
import pytest
from pydantic import ValidationError
from engine.types.operation import Operation
from engine.types.snapshot import Snapshot
from server.types.sync_request import SyncRequest
from server.types.sync_response import SyncResponse
# ---------------------------------------------------------------------------
# Operation
# ---------------------------------------------------------------------------
class TestOperation:
def _full_op(self) -> Operation:
return Operation(
id="op-1",
type="insert",
device_id="dev-1",
timestamp="1000000.0",
entity_id="entity-42",
entity_type="object",
payload={"key": "value", "count": 3},
)
def test_encode_returns_bytes(self) -> None:
assert isinstance(self._full_op().encode(), bytes)
def test_encode_decode_roundtrip(self) -> None:
op = self._full_op()
assert Operation.decode(op.encode()) == op
def test_all_fields_optional_allows_empty_instance(self) -> None:
op = Operation()
assert op.id is None
assert op.type is None
def test_decode_partial_json(self) -> None:
"""Only required fields in bytes → remaining fields default to None."""
raw = b'{"id": "x"}'
op = Operation.decode(raw)
assert op.id == "x"
assert op.type is None
def test_encode_is_utf8_json(self) -> None:
op = self._full_op()
parsed = json.loads(op.encode().decode("utf-8"))
assert parsed["id"] == "op-1"
assert parsed["payload"] == {"key": "value", "count": 3}
# ---------------------------------------------------------------------------
# Snapshot
# ---------------------------------------------------------------------------
class TestSnapshot:
def _full_snap(self) -> Snapshot:
return Snapshot(id="snap-1", cursor=10, data={"users": []}, timestamp=1000)
def test_encode_returns_bytes(self) -> None:
assert isinstance(self._full_snap().encode(), bytes)
def test_encode_decode_roundtrip(self) -> None:
snap = self._full_snap()
assert Snapshot.decode(snap.encode()) == snap
def test_all_fields_optional(self) -> None:
snap = Snapshot()
assert snap.id is None
assert snap.cursor is None
assert snap.data is None
assert snap.timestamp is None
# ---------------------------------------------------------------------------
# SyncRequest
# ---------------------------------------------------------------------------
class TestSyncRequest:
def _make(self, cursor: int = 0, ops: list[Operation] | None = None) -> SyncRequest:
return SyncRequest(
device_id="client-1",
cursor=cursor,
operations=ops or [],
)
def test_encode_returns_bytes(self) -> None:
assert isinstance(self._make().encode(), bytes)
def test_encode_decode_roundtrip_empty_ops(self) -> None:
req = self._make(cursor=5)
assert SyncRequest.decode(req.encode()) == req
def test_encode_decode_roundtrip_with_ops(self) -> None:
op = Operation(
id="1",
type="insert",
device_id="client-1",
timestamp="1.0",
entity_id="e1",
entity_type="obj",
payload={},
)
req = self._make(cursor=0, ops=[op])
decoded = SyncRequest.decode(req.encode())
assert decoded.device_id == "client-1"
assert len(decoded.operations) == 1
assert decoded.operations[0].id == "1"
def test_required_fields_enforced(self) -> None:
with pytest.raises(ValidationError):
SyncRequest(cursor=0, operations=[]) # type: ignore[call-arg]
def test_cursor_required(self) -> None:
with pytest.raises(ValidationError):
SyncRequest(device_id="x", operations=[]) # type: ignore[call-arg]
# ---------------------------------------------------------------------------
# SyncResponse
# ---------------------------------------------------------------------------
class TestSyncResponse:
def _make(
self,
cursor: int = 0,
ops: list[Operation] | None = None,
snapshot: Snapshot | None = None,
) -> SyncResponse:
return SyncResponse(cursor=cursor, operations=ops or [], snapshot=snapshot)
def test_encode_returns_bytes(self) -> None:
assert isinstance(self._make().encode(), bytes)
def test_encode_decode_roundtrip_no_snapshot(self) -> None:
res = self._make(cursor=3)
assert SyncResponse.decode(res.encode()) == res
def test_encode_decode_roundtrip_with_snapshot(self) -> None:
snap = Snapshot(id="s1", cursor=3, data={}, timestamp=999)
res = self._make(cursor=3, snapshot=snap)
decoded = SyncResponse.decode(res.encode())
assert decoded.snapshot is not None
assert decoded.snapshot.id == "s1"
def test_snapshot_defaults_to_none(self) -> None:
res = self._make()
assert res.snapshot is None
def test_operations_carries_nested_ops(self) -> None:
op = Operation(
id="2",
type="delete",
device_id="dev-1",
timestamp="2.0",
entity_id="e2",
entity_type="obj",
payload={},
)
res = self._make(cursor=1, ops=[op])
decoded = SyncResponse.decode(res.encode())
assert decoded.operations[0].type == "delete"