#!/usr/bin/env python3
"""Reversible profile changes; controls only explicitly owned systemd units."""
import argparse
import fcntl
import json
import os
import socket
import stat
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path('/home/lab/bench/ornith/modular')
PORTS = {'sglang': (8130, 8131), 'vllm': (8140, 8141)}


class Error(RuntimeError):
    pass


def atomic_json(path, value):
    path = Path(path)
    fd, name = tempfile.mkstemp(prefix='.state-', dir=path.parent)
    try:
        with os.fdopen(fd, 'w') as file:
            json.dump(value, file, indent=2)
            file.write('\n')
            file.flush()
            os.fsync(file.fileno())
        os.replace(name, path)
        directory = os.open(path.parent, os.O_RDONLY)
        try:
            os.fsync(directory)
        finally:
            os.close(directory)
    finally:
        if os.path.exists(name):
            os.unlink(name)


def read_json(path, default=None):
    try:
        return json.loads(Path(path).read_text())
    except FileNotFoundError:
        return default


def validate_profile(profile):
    if profile.get('status') != 'validated':
        raise Error('%s is %s: %s' % (profile.get('id'), profile.get('status'), profile.get('reason', 'not validated')))
    if not isinstance(profile.get('evidence'), list) or not profile['evidence']:
        raise Error('Validated profile has no evidence; refusing activation')
    command = profile.get('command')
    if not isinstance(command, list) or len(command) < 2 or not all(isinstance(s, str) and '\x00' not in s and '\n' not in s for s in command) or command[0] not in ('/bin/bash', '/usr/bin/bash') or not Path(command[1]).is_absolute():
        raise Error('Profile command must be [absolute bash, absolute launcher, optional arguments]')
    if not isinstance(profile.get('env', {}), dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in profile.get('env', {}).items()):
        raise Error('Profile environment must contain strings')
    if not profile.get('model_id'):
        raise Error('Profile must identify model_id')
    timeout = profile.get('health_timeout', 600)
    if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 1 <= timeout <= 3600:
        raise Error('health_timeout must be between 1 and 3600 seconds')


class Gateway:
    def __init__(self, path):
        self.path = Path(path)

    def call(self, command):
        st = self.path.stat()
        if not stat.S_ISSOCK(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
            raise Error('Unsafe gateway control socket')
        with socket.socket(socket.AF_UNIX) as client:
            client.settimeout(5)
            client.connect(str(self.path))
            client.sendall(json.dumps({'command': command}).encode() + b'\n')
            data = client.makefile('rb').readline(4096)
        result = json.loads(data)
        if 'error' in result:
            raise Error(result['error'])
        return result

    def drain(self, timeout=600):
        self.call('drain')
        deadline = time.monotonic() + timeout
        while True:
            state = self.call('status')
            if state['active'] == 0:
                return True
            if time.monotonic() >= deadline:
                return False
            time.sleep(min(.25, max(0, deadline - time.monotonic())))

    def resume(self):
        self.call('resume')


class Systemd:
    def __init__(self, engine, gpu_uuid, root=ROOT):
        self.engine, self.gpu_uuid, self.root = engine, gpu_uuid, Path(root)
        self.runtime = Path(os.environ.get('XDG_RUNTIME_DIR', '/run/user/%d' % os.getuid())) / 'ornithctl'
        self.socket = self.runtime / (engine + '.sock')

    def unit(self, gateway=False):
        return 'ornithctl-' + self.engine + ('-gateway' if gateway else '') + '.service'

    def marker(self, gateway=False):
        return 'Ornith modular owned %s %s' % (self.engine, 'gateway' if gateway else 'worker')

    def info(self, gateway=False):
        result = subprocess.run(['systemctl', '--user', 'show', self.unit(gateway), '--property=LoadState,ActiveState,Transient,Description,MainPID'], capture_output=True, text=True)
        if result.returncode:
            raise Error('Cannot inspect user systemd; refusing changes')
        return dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)

    def owned(self, gateway=False):
        info = self.info(gateway)
        if info.get('LoadState') == 'not-found':
            return False
        if info.get('Transient') != 'yes' or info.get('Description') != self.marker(gateway):
            raise Error('Unit name is occupied by an unowned unit; refusing to stop or replace it')
        return info.get('ActiveState') in ('active', 'activating', 'deactivating')

    def stop(self):
        # No PID discovery and no pkill: only a verified owned unit's cgroup.
        if self.owned():
            result = subprocess.run(['systemctl', '--user', 'stop', self.unit()], capture_output=True)
            if result.returncode:
                raise Error('Owned worker could not be stopped')
        subprocess.run(['systemctl', '--user', 'reset-failed', self.unit()], capture_output=True)
        # --collect transient services unload asynchronously; do not race reuse
        # of the same unit name while systemd still has the old definition.
        for _ in range(50):
            info = self.info()
            if info.get('LoadState') == 'not-found':
                return
            if info.get('Transient') != 'yes' or info.get('Description') != self.marker():
                raise Error('Unit ownership changed during stop; refusing reuse')
            time.sleep(.1)
        raise Error('Stopped transient unit has not unloaded; refusing name reuse')

    def run_unit(self, command, env, gateway=False):
        if self.owned(gateway):
            raise Error('Owned unit still running; refusing duplicate launch')
        args = ['systemd-run', '--user', '--quiet', '--collect', '--unit=' + self.unit(gateway),
                '--description=' + self.marker(gateway), '--service-type=exec',
                '--property=TimeoutStopSec=45', '--property=KillMode=control-group',
                '--property=Restart=no', '--property=UMask=0077']
        # An env -i ExecStart boundary prevents inheriting tokens/private values
        # from either this shell or the long-running user systemd manager.
        clean_env = {'HOME': str(Path.home()), 'USER': os.environ.get('USER', 'lab'),
                     'LOGNAME': os.environ.get('LOGNAME', 'lab'),
                     'PATH': '/home/lab/.local/bin:/usr/local/bin:/usr/bin:/bin',
                     'LANG': 'C.UTF-8', 'XDG_RUNTIME_DIR': str(self.runtime.parent)}
        clean_env.update(env)
        for key, value in clean_env.items():
            if not key.replace('_', '').isalnum() or '\x00' in value or '\n' in value:
                raise Error('Invalid environment entry')
        isolated = ['/usr/bin/env', '-i'] + [key + '=' + value for key, value in clean_env.items()] + command
        result = subprocess.run(args + ['--'] + isolated, capture_output=True)
        if result.returncode:
            raise Error('systemd refused owned service startup (see unit journal)')

    def start(self, profile):
        validate_profile(profile)
        launcher = Path(profile['command'][1])
        if not launcher.is_file():
            raise Error('Launcher is missing')
        with socket.socket() as check:
            check.settimeout(.5)
            if check.connect_ex(('127.0.0.1', PORTS[self.engine][1])) == 0:
                raise Error('Backend port already occupied; refusing to adopt or stop an unidentified process')
        env = dict(profile.get('env', {}))
        env.update(CUDA_VISIBLE_DEVICES=self.gpu_uuid, CUDA_DEVICE_ORDER='PCI_BUS_ID', ORNITH_PORT=str(PORTS[self.engine][1]))
        self.run_unit(profile['command'], env)

    def ensure_gateway(self):
        if self.owned(True):
            return
        self.runtime.mkdir(mode=0o700, parents=True, exist_ok=True)
        st = self.runtime.stat()
        if st.st_uid != os.getuid() or st.st_mode & 0o077:
            raise Error('Unsafe runtime directory')
        if self.socket.exists():
            # An untracked listener may own the socket: do not unlink it.
            try:
                Gateway(self.socket).call('status')
            except ConnectionRefusedError:
                if self.socket.lstat().st_uid != os.getuid() or not stat.S_ISSOCK(self.socket.lstat().st_mode):
                    raise Error('Unsafe stale socket')
                self.socket.unlink()
            else:
                raise Error('Untracked gateway socket is alive; refusing takeover')
        port, upstream = PORTS[self.engine]
        self.run_unit([sys.executable, str(self.root / 'control/ornith_gateway.py'), '--port', str(port), '--upstream-port', str(upstream), '--socket', str(self.socket)], {}, True)
        for _ in range(100):
            try:
                Gateway(self.socket).call('status')
                return
            except (OSError, ValueError):
                time.sleep(.1)
        raise Error('Gateway did not become ready')


def smoke(engine, profile, timeout=None):
    timeout = profile.get('health_timeout', 600) if timeout is None else timeout
    url = 'http://127.0.0.1:%d' % PORTS[engine][1]
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        try:
            with urllib.request.urlopen(url + '/health', timeout=5) as response:
                if response.status == 200:
                    break
        except (OSError, urllib.error.URLError):
            time.sleep(1)
    else:
        raise Error('Backend health timeout')
    request = urllib.request.Request(url + '/v1/completions', data=json.dumps({
        'model': profile['model_id'], 'prompt': 'Complete: 2 + 2 =', 'max_tokens': 16,
        'temperature': 0, 'stream': False}).encode(), headers={'Content-Type': 'application/json'})
    with urllib.request.urlopen(request, timeout=90) as response:
        answer = json.load(response)
    if not answer.get('choices') or not isinstance(answer['choices'][0].get('text'), str) or not answer['choices'][0]['text'].strip():
        raise Error('Bounded completion smoke failed')


class Controller:
    def __init__(self, engine, root=ROOT, service=None, gateway=None, probe=None, drain_timeout=600):
        self.engine, self.root = engine, Path(root)
        self.registry = read_json(self.root / 'profiles' / (engine + '.json'))
        if not self.registry or self.registry.get('engine') != engine:
            raise Error('Missing or invalid engine registry')
        self.profiles = {p['id']: p for p in self.registry['profiles']}
        if len(self.profiles) != len(self.registry['profiles']):
            raise Error('Duplicate profile IDs')
        self.path = self.root / 'state' / (engine + '.json')
        self.service = service or Systemd(engine, self.registry['gpu_uuid'], root)
        self.gateway = gateway or Gateway(self.service.socket)
        self.probe = probe or (lambda p: smoke(engine, p))
        self.drain_timeout = drain_timeout

    def status(self):
        result = {'engine': self.engine, 'state': read_json(self.path, {}), 'worker': self.service.info(), 'gateway': self.service.info(True)}
        try:
            result['admission'] = self.gateway.call('status')
        except (OSError, ValueError, Error):
            result['admission'] = {'available': False}
        # Never expose stored profile snapshots/environment.
        result['state'] = {k: v for k, v in result['state'].items() if k not in ('current_profile', 'previous_profile')}
        return result

    def listing(self):
        return [{'id': p['id'], 'status': p['status'], 'reason': p.get('reason', ''),
                 'context': p.get('context'), 'concurrency': p.get('concurrency')}
                for p in self.profiles.values()]

    def switch(self, target=None, rollback=False):
        self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
        with open(self.path.with_suffix('.lock'), 'a') as lock:
            os.chmod(lock.name, 0o600)
            try:
                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
            except BlockingIOError:
                raise Error('Another profile change is already in progress')
            state = read_json(self.path, {})
            if state.get('phase') not in (None, 'ready', 'failed-empty'):
                raise Error('Interrupted/failed transaction requires operator recovery; admission not changed')
            previous = state.get('current_profile')
            if rollback:
                candidate = state.get('previous_profile')
                if not candidate:
                    raise Error('No successful prior profile to roll back to')
            else:
                candidate = self.profiles.get(target)
                if not candidate:
                    raise Error('Unknown profile: ' + str(target))
            validate_profile(candidate)
            if previous:
                validate_profile(previous)
            if previous == candidate:
                return {'result': 'already-current', 'profile': candidate['id']}
            owned = self.service.owned()
            if previous and not owned:
                raise Error('Recorded worker is not active; refusing automatic replacement')
            if owned and not previous:
                raise Error('Worker lacks saved ownership state; operator migration required')
            if previous:
                if not self.service.owned(True):
                    raise Error('Recorded gateway is not active; refusing change')
                try:
                    if not self.gateway.drain(self.drain_timeout):
                        raise Error('Drain timed out; old worker remains running')
                except Exception:
                    self.gateway.resume()
                    raise
            changing = dict(state, phase='changing', attempted=candidate['id'])
            atomic_json(self.path, changing)
            try:
                if previous:
                    self.service.stop()
                self.service.start(candidate)
                self.probe(candidate)
                if not previous:
                    self.service.ensure_gateway()
                updated = {'phase': 'ready', 'current': candidate['id'], 'previous': previous['id'] if previous else None,
                           'current_profile': candidate, 'previous_profile': previous, 'updated_at': time.time()}
                atomic_json(self.path, updated)
                self.gateway.resume()
                return {'result': 'activated', 'profile': candidate['id']}
            except Exception as failure:
                # Failed candidate is stopped ONLY through owned-unit validation.
                try:
                    self.service.stop()
                    if previous:
                        self.service.start(previous)
                        self.probe(previous)
                        atomic_json(self.path, dict(state, phase='ready', last_failed=candidate['id']))
                        self.gateway.resume()
                        raise Error('Candidate failed; previous profile restored: ' + previous['id']) from failure
                    atomic_json(self.path, {'phase': 'failed-empty', 'last_failed': candidate['id']})
                except Error as rollback_error:
                    if str(rollback_error).startswith('Candidate failed; previous profile restored:'):
                        raise
                    atomic_json(self.path, dict(changing, phase='recovery-required'))
                    raise Error('Candidate and recovery failed; admission remains closed; inspect owned unit journal') from rollback_error
                except Exception as rollback_error:
                    atomic_json(self.path, dict(changing, phase='recovery-required'))
                    raise Error('Candidate and recovery failed; admission remains closed') from rollback_error
                raise Error('Initial activation failed; no prior worker existed') from failure


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('engine', choices=PORTS)
    parser.add_argument('action', choices=['list', 'status', 'use', 'rollback'])
    parser.add_argument('profile', nargs='?')
    args = parser.parse_args()
    try:
        controller = Controller(args.engine)
        if args.action == 'list':
            result = controller.listing()
        elif args.action == 'status':
            result = controller.status()
        elif args.action == 'use':
            if not args.profile:
                raise Error('use requires a profile ID')
            result = controller.switch(args.profile)
        else:
            result = controller.switch(rollback=True)
        print(json.dumps(result, indent=2))
    except (Error, OSError, ValueError) as error:
        print('ornithctl: ' + str(error), file=sys.stderr)
        return 1
    return 0


if __name__ == '__main__':
    sys.exit(main())
