Skip to content

Commit 6cee32f

Browse files
author
Paweł Kędzia
committed
**Refactor:** update _step3_unused_host to skip hosts already running the requested model and ensure the host is free by checking the Redis model‑hosts set.
1 parent 5958862 commit 6cee32f

File tree

1 file changed

+20
-7
lines changed

1 file changed

+20
-7
lines changed

llm_router_api/core/lb/strategies/first_available_optim.py

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -158,24 +158,37 @@ def _step2_existing_hosts(
158158
continue
159159
return None
160160

161-
# -------------------------------------------------------------------------
162-
# Step 3 – pick an entirely unused host
163-
# -------------------------------------------------------------------------
161+
# ---------------------------------------------------------------------
162+
# Step 3 – pick a host that does NOT already have this model loaded
163+
# --------------------------------------------------------------------------
164164

165165
def _step3_unused_host(
166166
self, model_name: str, providers: List[Dict]
167167
) -> Optional[Dict]:
168168
"""
169-
Find a host that has never been used for any model (no occupancy entry)
170-
and acquire a provider on it.
169+
Find a host that does **not** already run ``model_name`` (i.e. the host
170+
is not present in the ``:hosts`` Redis set for this model) and acquire a
171+
provider on it. The host must also be free – it must not be occupied by
172+
another model.
171173
"""
174+
# Hosts that already have this model loaded.
175+
known_hosts_key = self._model_hosts_set_key(model_name)
176+
known_hosts_bytes = self.redis_client.smembers(known_hosts_key)
177+
known_hosts = {b for b in known_hosts_bytes} if known_hosts_bytes else set()
178+
172179
for provider in providers:
173180
host = self._host_from_provider(provider)
174181
if not host:
175182
continue
183+
184+
# Skip hosts that already run the requested model.
185+
if host in known_hosts:
186+
continue
187+
188+
# Ensure the host is not currently occupied by a different model.
176189
occ_key = self._host_occupancy_key(host)
177-
if self.redis_client.hexists(occ_key, "model"):
178-
# Host already assigned to some model.
190+
current_model = self.redis_client.hget(occ_key, "model")
191+
if current_model and current_model != model_name:
179192
continue
180193

181194
field = self._provider_field(provider)

0 commit comments

Comments
 (0)