| 1 | using System.Diagnostics; |
| 2 | using CascadeIDE.Cockpit.DataBus; |
| 3 | using CascadeIDE.Features.Agent.Environment; |
| 4 | using CascadeIDE.Models; |
| 5 | using Xunit; |
| 6 | |
| 7 | namespace CascadeIDE.Tests; |
| 8 | |
| 9 | /// <summary> |
| 10 | /// Orion local test-drive scenarios (ADR 0148): stale/coalesce, substrate isolation, host death. |
| 11 | /// </summary> |
| 12 | [Trait("Category", "AgentEnvironment")] |
| 13 | public sealed class AgentEnvironmentOrionStressTests |
| 14 | { |
| 15 | private static string? TryResolveCascadeSolutionPath() |
| 16 | { |
| 17 | var dir = AppContext.BaseDirectory; |
| 18 | for (var i = 0; i < 8; i++) |
| 19 | { |
| 20 | var candidate = Path.Combine(dir, "CascadeIDE.sln"); |
| 21 | if (File.Exists(candidate)) |
| 22 | return candidate; |
| 23 | |
| 24 | var parent = Directory.GetParent(dir); |
| 25 | if (parent is null) |
| 26 | break; |
| 27 | dir = parent.FullName; |
| 28 | } |
| 29 | |
| 30 | return null; |
| 31 | } |
| 32 | |
| 33 | [Fact] |
| 34 | public void Stress_Dedup_CoalescesBuildKeyWithin1500ms() |
| 35 | { |
| 36 | var dedup = new EnvironmentTaskDedup(1500); |
| 37 | var key = "build|solution.sln"; |
| 38 | |
| 39 | Assert.False(dedup.ShouldCoalesce(key)); |
| 40 | Assert.True(dedup.ShouldCoalesce(key)); |
| 41 | |
| 42 | Thread.Sleep(1600); |
| 43 | Assert.False(dedup.ShouldCoalesce(key)); |
| 44 | } |
| 45 | |
| 46 | [Fact] |
| 47 | public async Task Stress_MicroWrites_EmitsEpochStaleDuringActiveVerify() |
| 48 | { |
| 49 | var bus = new InMemoryDataBus(); |
| 50 | var staleEvents = new List<AgentVerifyEpochStale>(); |
| 51 | bus.Subscribe<AgentVerifyEpochStale>(e => staleEvents.Add(e)); |
| 52 | |
| 53 | var tracker = new AgentVerifyEpochTracker(bus); |
| 54 | tracker.Begin("run-a", "snap-1", @"C:\repo\app.sln"); |
| 55 | |
| 56 | for (var i = 0; i < 20; i++) |
| 57 | { |
| 58 | tracker.NotifyWrite(@"C:\repo\src\File" + i + ".cs"); |
| 59 | await Task.Delay(50); |
| 60 | } |
| 61 | |
| 62 | Assert.True(staleEvents.Count >= 15); |
| 63 | Assert.All(staleEvents, e => Assert.Equal("write_in_epoch", e.Reason)); |
| 64 | } |
| 65 | |
| 66 | [Fact] |
| 67 | public async Task Stress_RapidVerify_SupersedesPredecessor_NotParallelActive() |
| 68 | { |
| 69 | var sln = TryResolveCascadeSolutionPath(); |
| 70 | if (sln is null) |
| 71 | return; |
| 72 | |
| 73 | var bus = new InMemoryDataBus(); |
| 74 | var superseded = 0; |
| 75 | bus.Subscribe<AgentVerifyEpochStale>(e => |
| 76 | { |
| 77 | if (e.Reason == "superseded") |
| 78 | Interlocked.Increment(ref superseded); |
| 79 | }); |
| 80 | |
| 81 | var settings = new AgentEnvironmentSettings { CoalesceWindowMs = 1500 }; |
| 82 | var svc = new AgentEnvironmentService(bus, settings); |
| 83 | |
| 84 | var sawActiveBeforeSecondStart = false; |
| 85 | for (var i = 0; i < 15; i++) |
| 86 | { |
| 87 | var wasActive = svc.GetStatus().IsActive; |
| 88 | var start = svc.StartVerify(sln, AgentVerifyPolicy.Minimal); |
| 89 | Assert.True(start.Accepted); |
| 90 | Assert.True(svc.GetStatus().IsActive); |
| 91 | if (i > 0 && wasActive) |
| 92 | sawActiveBeforeSecondStart = true; |
| 93 | await Task.Delay(50); |
| 94 | } |
| 95 | |
| 96 | Assert.True( |
| 97 | superseded >= 1 || sawActiveBeforeSecondStart, |
| 98 | $"rapid verify should overlap or emit superseded; superseded={superseded}, overlap={sawActiveBeforeSecondStart}"); |
| 99 | |
| 100 | svc.CancelActive(); |
| 101 | await Task.Delay(100); |
| 102 | Assert.False(svc.GetStatus().IsActive); |
| 103 | } |
| 104 | |
| 105 | [Fact] |
| 106 | public async Task Stress_ParallelSubstrate_PortsAndDbDoNotLeak() |
| 107 | { |
| 108 | var root = Path.Combine(Path.GetTempPath(), "cide-orion-" + Guid.NewGuid().ToString("N")); |
| 109 | var manager = new AgentSandboxManager(Path.Combine(root, "agent-runs")); |
| 110 | |
| 111 | try |
| 112 | { |
| 113 | var runs = await Task.WhenAll(Enumerable.Range(0, 3).Select(async i => |
| 114 | { |
| 115 | var runId = $"parallel-{i}"; |
| 116 | var lease = manager.Prepare(runId, AgentSandboxProfile.AgentEphemeral); |
| 117 | var bundle = lease.Substrate ?? throw new InvalidOperationException("substrate missing"); |
| 118 | var payload = $"owner-{runId}"; |
| 119 | await Task.Run(() => |
| 120 | { |
| 121 | AgentSandboxSubstrate.WriteHeavyMarker(bundle, payload); |
| 122 | Thread.Sleep(200); |
| 123 | }); |
| 124 | return (runId, bundle, payload); |
| 125 | })); |
| 126 | |
| 127 | var ports = runs.Select(r => r.bundle.DevPort).ToArray(); |
| 128 | Assert.Equal(ports.Length, ports.Distinct().Count()); |
| 129 | |
| 130 | foreach (var (runId, bundle, payload) in runs) |
| 131 | { |
| 132 | var dbOwner = AgentSandboxSubstrate.ReadDatabaseOwner(bundle); |
| 133 | Assert.Equal(payload, dbOwner); |
| 134 | Assert.Contains(runId, File.ReadAllText(bundle.MarkerPath)); |
| 135 | } |
| 136 | |
| 137 | var secondWave = runs.Select(r => manager.RecreateSubstrateBeforeTests( |
| 138 | new AgentSandboxLease(r.runId, AgentSandboxProfile.AgentEphemeral, Path.Combine(manager.RunsRoot, r.runId), null, r.bundle))).ToList(); |
| 139 | |
| 140 | Assert.Equal(3, secondWave.Select(b => b.DevPort).Distinct().Count()); |
| 141 | } |
| 142 | finally |
| 143 | { |
| 144 | try |
| 145 | { |
| 146 | Directory.Delete(root, recursive: true); |
| 147 | } |
| 148 | catch |
| 149 | { |
| 150 | // best-effort |
| 151 | } |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | [Fact] |
| 156 | public async Task Stress_HostDeath_PublishesDiedQuickly() |
| 157 | { |
| 158 | var bus = new InMemoryDataBus(); |
| 159 | AgentEnvironmentTaskDied? died = null; |
| 160 | var diedAtMs = -1L; |
| 161 | var sw = Stopwatch.StartNew(); |
| 162 | bus.Subscribe<AgentEnvironmentTaskDied>(e => |
| 163 | { |
| 164 | died = e; |
| 165 | diedAtMs = sw.ElapsedMilliseconds; |
| 166 | }); |
| 167 | |
| 168 | var coordinator = new DotNetBuildTest.Core.BuildTestJobCoordinator(); |
| 169 | var runner = new EnvironmentTaskRunner(bus, coordinator) |
| 170 | { |
| 171 | TestJobStatusFactory = _ => null, |
| 172 | }; |
| 173 | |
| 174 | var sln = TryResolveCascadeSolutionPath() |
| 175 | ?? throw new InvalidOperationException("CascadeIDE.sln not found for host-death stress test."); |
| 176 | |
| 177 | var outcome = await runner.RunBuildAsync("run-death", sln, waitForCompletion: false); |
| 178 | Assert.True( |
| 179 | outcome.CoreJobId is not null || outcome.Status is "queued", |
| 180 | $"build task not enqueued: {outcome.Status}"); |
| 181 | |
| 182 | for (var i = 0; i < 30 && died is null; i++) |
| 183 | await Task.Delay(50); |
| 184 | |
| 185 | Assert.NotNull(died); |
| 186 | Assert.True(diedAtMs < 2000, $"Died event too slow: {diedAtMs}ms"); |
| 187 | Assert.Equal("supervised-inproc", died!.HostKind); |
| 188 | } |
| 189 | |
| 190 | [Fact] |
| 191 | public async Task Stress_RapidVerifyAndMicroWrites_NoUnhandledException() |
| 192 | { |
| 193 | var sln = TryResolveCascadeSolutionPath(); |
| 194 | if (sln is null) |
| 195 | return; |
| 196 | |
| 197 | var bus = new InMemoryDataBus(); |
| 198 | var svc = new AgentEnvironmentService(bus, new AgentEnvironmentSettings { CoalesceWindowMs = 1500 }); |
| 199 | var scratch = Path.Combine(Path.GetTempPath(), "cide-scratch-" + Guid.NewGuid().ToString("N") + ".cs"); |
| 200 | |
| 201 | try |
| 202 | { |
| 203 | await File.WriteAllTextAsync(scratch, "// scratch"); |
| 204 | var writeTask = Task.Run(async () => |
| 205 | { |
| 206 | for (var i = 0; i < 30; i++) |
| 207 | { |
| 208 | await File.AppendAllTextAsync(scratch, "\n// " + i); |
| 209 | svc.EpochTracker.NotifyWrite(scratch); |
| 210 | await Task.Delay(500); |
| 211 | } |
| 212 | }); |
| 213 | |
| 214 | var verifyTask = Task.Run(async () => |
| 215 | { |
| 216 | for (var i = 0; i < 10; i++) |
| 217 | { |
| 218 | svc.StartVerify(sln, AgentVerifyPolicy.Minimal); |
| 219 | await Task.Delay(500); |
| 220 | } |
| 221 | }); |
| 222 | |
| 223 | await Task.WhenAll(writeTask, verifyTask); |
| 224 | } |
| 225 | finally |
| 226 | { |
| 227 | try |
| 228 | { |
| 229 | File.Delete(scratch); |
| 230 | } |
| 231 | catch |
| 232 | { |
| 233 | // best-effort |
| 234 | } |
| 235 | |
| 236 | svc.CancelActive(); |
| 237 | } |
| 238 | } |
| 239 | } |
| 240 | |