#!/usr/bin/env python3
"""Reversible profile changes; controls only explicitly owned systemd units."""
import argparse
import fcntl
import json
import os
import socket
import stat
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path('/home/lab/bench/ornith/modular')
PORTS = {'sglang': (8130, 8131), 'vllm': (8140, 8141)}


class Error(RuntimeError):
    pass


def atomic_json(path, value):
    path = Path(path)
    fd, name = tempfile.mkstemp(prefix='.state-', dir=path.parent)
    try:
        with os.fdopen(fd, 'w') as file:
            json.dump(value, file, indent=2)
            file.write('\n')
            file.flush()
            os.fsync(file.fileno())
        os.replace(name, path)
        directory = os.open(path.parent, os.O_RDONLY)
        try:
            os.fsync(directory)
        finally:
            os.close(directory)
    finally:
        if os.path.exists(name):
            os.unlink(name)


def read_json(path, default=None):
    try:
        return json.loads(Path(path).read_text())
    except FileNotFoundError:
        return default


def validate_profile(profile):
    if profile.get('status') != 'validated':
        raise Error('%s is %s: %s' % (profile.get('id'), profile.get('status'), profile.get('reason', 'not validated')))
    if not isinstance(profile.get('evidence'), list) or not profile['evidence']:
        raise Error('Validated profile has no evidence; refusing activation')
    command = profile.get('command')
    if not isinstance(command, list) or len(command) < 2 or not all(isinstance(s, str) and '\x00' not in s and '\n' not in s for s in command) or command[0] not in ('/bin/bash', '/usr/bin/bash') or not Path(command[1]).is_absolute():
        raise Error('Profile command must be [absolute bash, absolute launcher, optional arguments]')
    if not isinstance(profile.get('env', {}), dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in profile.get('env', {}).items()):
        raise Error('Profile environment must contain strings')
    if not profile.get('model_id'):
        raise Error('Profile must identify model_id')
    timeout = profile.get('health_timeout', 600)
    if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 1 <= timeout <= 3600:
        raise Error('health_timeout must be between 1 and 3600 seconds')
    power = profile.get('max_power_limit_w')
    if power is not None and (isinstance(power, bool) or not isinstance(power, (int, float)) or not 1 <= power <= 1000):
        raise Error('max_power_limit_w must be between 1 and 1000 watts')


class Gateway:
    def __init__(self, path):
        self.path = Path(path)

    def call(self, command):
        st = self.path.stat()
        if not stat.S_ISSOCK(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
            raise Error('Unsafe gateway control socket')
        with socket.socket(socket.AF_UNIX) as client:
            client.settimeout(5)
            client.connect(str(self.path))
            client.sendall(json.dumps({'command': command}).encode() + b'\n')
            data = client.makefile('rb').readline(4096)
        result = json.loads(data)
        if 'error' in result:
            raise Error(result['error'])
        return result

    def drain(self, timeout=600):
        self.call('drain')
        deadline = time.monotonic() + timeout
        while True:
            state = self.call('status')
            if state['active'] == 0:
                return True
            if time.monotonic() >= deadline:
                return False
            time.sleep(min(.25, max(0, deadline - time.monotonic())))

    def resume(self):
        self.call('resume')


class Systemd:
    def __init__(self, engine, gpu_uuid, root=ROOT):
        self.engine, self.gpu_uuid, self.root = engine, gpu_uuid, Path(root)
        self.runtime = Path(os.environ.get('XDG_RUNTIME_DIR', '/run/user/%d' % os.getuid())) / 'ornithctl'
        self.socket = self.runtime / (engine + '.sock')

    def unit(self, gateway=False):
        return 'ornithctl-' + self.engine + ('-gateway' if gateway else '') + '.service'

    def check_safety(self, profile=None):
        latch = self.root / self.engine / 'thermal-stop.json'
        if latch.exists():
            raise Error('Thermal safety latch present; manual review required before any worker start: ' + str(latch))
        limit = (profile or {}).get('max_power_limit_w')
        if limit is not None:
            try:
                current = float(subprocess.check_output([
                    '/usr/bin/nvidia-smi', '-i', self.gpu_uuid, '--query-gpu=power.limit',
                    '--format=csv,noheader,nounits'], text=True, timeout=10).strip())
            except (OSError, ValueError, subprocess.SubprocessError) as exc:
                raise Error('Cannot verify required GPU power limit; refusing worker start') from exc
            if not 0 < current <= limit + .5:
                raise Error('GPU power limit is %s W; profile requires at most %s W. No hardware setting was changed.' % (current, limit))

    def marker(self, gateway=False):
        return 'Ornith modular owned %s %s' % (self.engine, 'gateway' if gateway else 'worker')

    def info(self, gateway=False):
        result = subprocess.run(['systemctl', '--user', 'show', self.unit(gateway), '--property=LoadState,ActiveState,Transient,Description,MainPID'], capture_output=True, text=True)
        if result.returncode:
            raise Error('Cannot inspect user systemd; refusing changes')
        return dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)

    def owned(self, gateway=False):
        info = self.info(gateway)
        if info.get('LoadState') == 'not-found':
            return False
        if info.get('Transient') != 'yes' or info.get('Description') != self.marker(gateway):
            raise Error('Unit name is occupied by an unowned unit; refusing to stop or replace it')
        return info.get('ActiveState') in ('active', 'activating', 'deactivating')

    def stop(self):
        # No PID discovery and no pkill: only a verified owned unit's cgroup.
        if self.owned():
            result = subprocess.run(['systemctl', '--user', 'stop', self.unit()], capture_output=True)
            if result.returncode:
                raise Error('Owned worker could not be stopped')
        subprocess.run(['systemctl', '--user', 'reset-failed', self.unit()], capture_output=True)
        # --collect transient services unload asynchronously; do not race reuse
        # of the same unit name while systemd still has the old definition.
        for _ in range(50):
            info = self.info()
            if info.get('LoadState') == 'not-found':
                return
            if info.get('Transient') != 'yes' or info.get('Description') != self.marker():
                raise Error('Unit ownership changed during stop; refusing reuse')
            time.sleep(.1)
        raise Error('Stopped transient unit has not unloaded; refusing name reuse')

    def run_unit(self, command, env, gateway=False):
        if self.owned(gateway):
            raise Error('Owned unit still running; refusing duplicate launch')
        args = ['systemd-run', '--user', '--quiet', '--collect', '--unit=' + self.unit(gateway),
                '--description=' + self.marker(gateway), '--service-type=exec',
                '--property=TimeoutStopSec=45', '--property=KillMode=control-group',
                '--property=Restart=no', '--property=UMask=0077']
        # An env -i ExecStart boundary prevents inheriting tokens/private values
        # from either this shell or the long-running user systemd manager.
        clean_env = {'HOME': str(Path.home()), 'USER': os.environ.get('USER', 'lab'),
                     'LOGNAME': os.environ.get('LOGNAME', 'lab'),
                     'PATH': '/home/lab/.local/bin:/usr/local/bin:/usr/bin:/bin',
                     'LANG': 'C.UTF-8', 'XDG_RUNTIME_DIR': str(self.runtime.parent)}
        clean_env.update(env)
        for key, value in clean_env.items():
            if not key.replace('_', '').isalnum() or '\x00' in value or '\n' in value:
                raise Error('Invalid environment entry')
        isolated = ['/usr/bin/env', '-i'] + [key + '=' + value for key, value in clean_env.items()] + command
        result = subprocess.run(args + ['--'] + isolated, capture_output=True)
        if result.returncode:
            raise Error('systemd refused owned service startup (see unit journal)')

    def start(self, profile):
        validate_profile(profile)
        self.check_safety(profile)
        launcher = Path(profile['command'][1])
        if not launcher.is_file():
            raise Error('Launcher is missing')
        with socket.socket() as check:
            check.settimeout(.5)
            if check.connect_ex(('127.0.0.1', PORTS[self.engine][1])) == 0:
                raise Error('Backend port already occupied; refusing to adopt or stop an unidentified process')
        if not self.owned(True):
            # Initial migration must free the old public listener BEFORE any
            # new GPU worker is launched, not only before gateway startup.
            with socket.socket() as check:
                check.settimeout(.5)
                if check.connect_ex(('127.0.0.1', PORTS[self.engine][0])) == 0:
                    raise Error('Public port already occupied by an unmanaged server; coordinate migration first')
        env = dict(profile.get('env', {}))
        env.update(CUDA_VISIBLE_DEVICES=self.gpu_uuid, CUDA_DEVICE_ORDER='PCI_BUS_ID', ORNITH_PORT=str(PORTS[self.engine][1]))
        self.run_unit(profile['command'], env)

    def ensure_gateway(self):
        if self.owned(True):
            return
        self.runtime.mkdir(mode=0o700, parents=True, exist_ok=True)
        st = self.runtime.stat()
        if st.st_uid != os.getuid() or st.st_mode & 0o077:
            raise Error('Unsafe runtime directory')
        if self.socket.exists():
            # An untracked listener may own the socket: do not unlink it.
            try:
                Gateway(self.socket).call('status')
            except ConnectionRefusedError:
                if self.socket.lstat().st_uid != os.getuid() or not stat.S_ISSOCK(self.socket.lstat().st_mode):
                    raise Error('Unsafe stale socket')
                self.socket.unlink()
            else:
                raise Error('Untracked gateway socket is alive; refusing takeover')
        port, upstream = PORTS[self.engine]
        self.run_unit([sys.executable, str(self.root / 'control/ornith_gateway.py'), '--port', str(port), '--upstream-port', str(upstream), '--socket', str(self.socket)], {}, True)
        for _ in range(100):
            try:
                Gateway(self.socket).call('status')
                return
            except (OSError, ValueError):
                time.sleep(.1)
        raise Error('Gateway did not become ready')


def smoke(engine, profile, timeout=None):
    timeout = profile.get('health_timeout', 600) if timeout is None else timeout
    url = 'http://127.0.0.1:%d' % PORTS[engine][1]
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        try:
            with urllib.request.urlopen(url + '/health', timeout=5) as response:
                if response.status == 200:
                    break
        except (OSError, urllib.error.URLError):
            time.sleep(1)
    else:
        raise Error('Backend health timeout')
    request = urllib.request.Request(url + '/v1/completions', data=json.dumps({
        'model': profile['model_id'], 'prompt': 'Complete: 2 + 2 =', 'max_tokens': 16,
        'temperature': 0, 'stream': False}).encode(), headers={'Content-Type': 'application/json'})
    with urllib.request.urlopen(request, timeout=90) as response:
        answer = json.load(response)
    if not answer.get('choices') or not isinstance(answer['choices'][0].get('text'), str) or not answer['choices'][0]['text'].strip():
        raise Error('Bounded completion smoke failed')


class Controller:
    def __init__(self, engine, root=ROOT, service=None, gateway=None, probe=None, drain_timeout=600):
        self.engine, self.root = engine, Path(root)
        self.registry = read_json(self.root / 'profiles' / (engine + '.json'))
        if not self.registry or self.registry.get('engine') != engine:
            raise Error('Missing or invalid engine registry')
        self.profiles = {p['id']: p for p in self.registry['profiles']}
        if len(self.profiles) != len(self.registry['profiles']):
            raise Error('Duplicate profile IDs')
        self.path = self.root / 'state' / (engine + '.json')
        self.service = service or Systemd(engine, self.registry['gpu_uuid'], root)
        self.gateway = gateway or Gateway(self.service.socket)
        self.probe = probe or (lambda p: smoke(engine, p))
        self.drain_timeout = drain_timeout

    def status(self):
        result = {'engine': self.engine, 'state': read_json(self.path, {}), 'worker': self.service.info(), 'gateway': self.service.info(True)}
        try:
            result['admission'] = self.gateway.call('status')
        except (OSError, ValueError, Error):
            result['admission'] = {'available': False}
        # Never expose stored profile snapshots/environment.
        result['state'] = {k: v for k, v in result['state'].items() if k not in ('current_profile', 'previous_profile')}
        return result

    def listing(self):
        return [{'id': p['id'], 'status': p['status'], 'reason': p.get('reason', ''),
                 'context': p.get('context'), 'concurrency': p.get('concurrency'),
                 'max_power_limit_w': p.get('max_power_limit_w')}
                for p in self.profiles.values()]

    def switch(self, target=None, rollback=False):
        self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
        with open(self.path.with_suffix('.lock'), 'a') as lock:
            os.chmod(lock.name, 0o600)
            try:
                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
            except BlockingIOError:
                raise Error('Another profile change is already in progress')
            state = read_json(self.path, {})
            if state.get('phase') not in (None, 'ready', 'failed-empty'):
                raise Error('Interrupted/failed transaction requires operator recovery; admission not changed')
            previous = state.get('current_profile')
            if rollback:
                candidate = state.get('previous_profile')
                if not candidate:
                    raise Error('No successful prior profile to roll back to')
            else:
                candidate = self.profiles.get(target)
                if not candidate:
                    raise Error('Unknown profile: ' + str(target))
            validate_profile(candidate)
            if previous:
                validate_profile(previous)
            self.service.check_safety(candidate)
            owned = self.service.owned()
            if previous and not owned:
                raise Error('Recorded worker is not active; refusing automatic replacement')
            if owned and not previous:
                raise Error('Worker lacks saved ownership state; operator migration required')
            if previous:
                if not self.service.owned(True):
                    raise Error('Recorded gateway is not active; refusing change')
            if previous == candidate:
                if self.gateway.call('status').get('draining', True):
                    raise Error('Current gateway is draining; operator review required')
                self.probe(candidate)
                return {'result': 'already-current', 'profile': candidate['id']}
            if previous:
                try:
                    if not self.gateway.drain(self.drain_timeout):
                        raise Error('Drain timed out; old worker remains running')
                except Exception:
                    self.gateway.resume()
                    raise
            changing = dict(state, phase='changing', attempted=candidate['id'])
            try:
                atomic_json(self.path, changing)
            except Exception as persistence_error:
                # No worker has been touched. Restore admission even when a
                # disk/fsync failure prevents saving the transaction marker.
                if previous:
                    try:
                        self.gateway.resume()
                    except Exception as admission_error:
                        raise Error('State persistence and admission recovery failed; previous worker unchanged; operator review required') from admission_error
                raise Error('Could not persist transaction; worker unchanged; inspect state before retrying') from persistence_error
            try:
                if previous:
                    self.service.stop()
                self.service.start(candidate)
                self.probe(candidate)
                if not previous:
                    self.service.ensure_gateway()
                updated = {'phase': 'ready', 'current': candidate['id'], 'previous': previous['id'] if previous else None,
                           'current_profile': candidate, 'previous_profile': previous, 'updated_at': time.time()}
                atomic_json(self.path, updated)
                self.gateway.resume()
                return {'result': 'activated', 'profile': candidate['id']}
            except Exception as failure:
                # Failed candidate is stopped ONLY through owned-unit validation.
                try:
                    self.service.stop()
                    if previous:
                        self.service.start(previous)
                        self.probe(previous)
                        atomic_json(self.path, dict(state, phase='ready', last_failed=candidate['id']))
                        self.gateway.resume()
                        raise Error('Candidate failed; previous profile restored: ' + previous['id']) from failure
                    atomic_json(self.path, {'phase': 'failed-empty', 'last_failed': candidate['id']})
                except Error as rollback_error:
                    if str(rollback_error).startswith('Candidate failed; previous profile restored:'):
                        raise
                    atomic_json(self.path, dict(changing, phase='recovery-required'))
                    raise Error('Candidate and recovery failed; admission remains closed; inspect owned unit journal') from rollback_error
                except Exception as rollback_error:
                    atomic_json(self.path, dict(changing, phase='recovery-required'))
                    raise Error('Candidate and recovery failed; admission remains closed') from rollback_error
                raise Error('Initial activation failed; no prior worker existed') from failure


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('engine', choices=PORTS)
    parser.add_argument('action', choices=['list', 'status', 'use', 'rollback'])
    parser.add_argument('profile', nargs='?')
    args = parser.parse_args()
    try:
        controller = Controller(args.engine)
        if args.action == 'list':
            result = controller.listing()
        elif args.action == 'status':
            result = controller.status()
        elif args.action == 'use':
            if not args.profile:
                raise Error('use requires a profile ID')
            result = controller.switch(args.profile)
        else:
            result = controller.switch(rollback=True)
        print(json.dumps(result, indent=2))
    except (Error, OSError, ValueError) as error:
        print('ornithctl: ' + str(error), file=sys.stderr)
        return 1
    return 0


if __name__ == '__main__':
    sys.exit(main())
