はじめに

これまでの記事で、Orleansの基本から高可用性システムの構築まで学んできました。今回は、実際の本番環境で直面する「パフォーマンスチューニングとトラブルシューティング」について解説します。

100万req/sを処理するシステムのボトルネック解析、メモリリークの特定と解決、デッドロックの回避...実際のプロジェクトで遭遇した問題と、その解決方法を共有します。

パフォーマンス測定の基礎

ベンチマークの実装

// BenchmarkGrain.cs
public interface IBenchmarkGrain : IGrainWithIntegerKey
{
    Task<BenchmarkResult> RunCpuBoundBenchmarkAsync(int iterations);
    Task<BenchmarkResult> RunIoBoundBenchmarkAsync(int operations);
    Task<BenchmarkResult> RunMemoryBenchmarkAsync(int allocations);
}

public class BenchmarkGrain : Grain, IBenchmarkGrain
{
    private readonly ILogger<BenchmarkGrain> _logger;
    
    public BenchmarkGrain(ILogger<BenchmarkGrain> logger)
    {
        _logger = logger;
    }
    
    public async Task<BenchmarkResult> RunCpuBoundBenchmarkAsync(int iterations)
    {
        var stopwatch = Stopwatch.StartNew();
        var startCpu = Process.GetCurrentProcess().TotalProcessorTime;
        
        // CPU集約的な処理
        var result = 0.0;
        for (int i = 0; i < iterations; i++)
        {
            result += Math.Sqrt(i) * Math.Sin(i) * Math.Cos(i);
        }
        
        stopwatch.Stop();
        var endCpu = Process.GetCurrentProcess().TotalProcessorTime;
        
        return new BenchmarkResult
        {
            ElapsedMilliseconds = stopwatch.ElapsedMilliseconds,
            CpuMilliseconds = (endCpu - startCpu).TotalMilliseconds,
            OperationsPerSecond = iterations / stopwatch.Elapsed.TotalSeconds,
            GrainId = this.GetPrimaryKeyLong()
        };
    }
    
    public async Task<BenchmarkResult> RunIoBoundBenchmarkAsync(int operations)
    {
        var stopwatch = Stopwatch.StartNew();
        var tasks = new List<Task>();
        
        for (int i = 0; i < operations; i++)
        {
            tasks.Add(SimulateIoOperationAsync());
        }
        
        await Task.WhenAll(tasks);
        stopwatch.Stop();
        
        return new BenchmarkResult
        {
            ElapsedMilliseconds = stopwatch.ElapsedMilliseconds,
            OperationsPerSecond = operations / stopwatch.Elapsed.TotalSeconds,
            GrainId = this.GetPrimaryKeyLong()
        };
    }
    
    private async Task SimulateIoOperationAsync()
    {
        // I/O操作のシミュレーション
        await Task.Delay(10);
    }
}

// 負荷テストクライアント
public class LoadTestClient
{
    private readonly IGrainFactory _grainFactory;
    private readonly ILogger<LoadTestClient> _logger;
    
    public async Task<LoadTestResult> RunLoadTestAsync(LoadTestOptions options)
    {
        var results = new ConcurrentBag<BenchmarkResult>();
        var errors = new ConcurrentBag<Exception>();
        var stopwatch = Stopwatch.StartNew();
        
        // 並列実行
        var tasks = Enumerable.Range(0, options.ConcurrentClients)
            .Select(async clientId =>
            {
                for (int i = 0; i < options.RequestsPerClient; i++)
                {
                    try
                    {
                        var grainId = clientId % options.GrainCount;
                        var grain = _grainFactory.GetGrain<IBenchmarkGrain>(grainId);
                        
                        var result = await grain.RunCpuBoundBenchmarkAsync(
                            options.IterationsPerRequest);
                        
                        results.Add(result);
                    }
                    catch (Exception ex)
                    {
                        errors.Add(ex);
                    }
                }
            });
        
        await Task.WhenAll(tasks);
        stopwatch.Stop();
        
        return new LoadTestResult
        {
            TotalRequests = options.ConcurrentClients * options.RequestsPerClient,
            SuccessfulRequests = results.Count,
            FailedRequests = errors.Count,
            TotalDurationMs = stopwatch.ElapsedMilliseconds,
            RequestsPerSecond = results.Count / stopwatch.Elapsed.TotalSeconds,
            AverageLatencyMs = results.Average(r => r.ElapsedMilliseconds),
            P95LatencyMs = results.OrderBy(r => r.ElapsedMilliseconds)
                .Skip((int)(results.Count * 0.95)).First().ElapsedMilliseconds,
            P99LatencyMs = results.OrderBy(r => r.ElapsedMilliseconds)
                .Skip((int)(results.Count * 0.99)).First().ElapsedMilliseconds
        };
    }
}

プロファイリングツールの活用

// カスタムプロファイラー
public class OrleansProfiler : IGrainCallFilter
{
    private readonly ILogger<OrleansProfiler> _logger;
    private readonly ActivitySource _activitySource;
    private readonly IMeterFactory _meterFactory;
    private readonly Histogram<double> _grainCallDuration;
    private readonly Counter<long> _grainCallCount;
    
    public OrleansProfiler(
        ILogger<OrleansProfiler> logger,
        IMeterFactory meterFactory)
    {
        _logger = logger;
        _activitySource = new ActivitySource("Orleans.Profiler");
        _meterFactory = meterFactory;
        
        var meter = meterFactory.Create("Orleans.Performance");
        _grainCallDuration = meter.CreateHistogram<double>(
            "orleans.grain.call.duration",
            unit: "ms",
            description: "Duration of grain method calls");
        
        _grainCallCount = meter.CreateCounter<long>(
            "orleans.grain.call.count",
            description: "Number of grain method calls");
    }
    
    public async Task Invoke(IIncomingGrainCallContext context)
    {
        var grainType = context.Grain.GetType().Name;
        var methodName = context.InterfaceMethod.Name;
        
        using var activity = _activitySource.StartActivity(
            $"{grainType}.{methodName}",
            ActivityKind.Internal);
        
        var stopwatch = Stopwatch.StartNew();
        
        try
        {
            await context.Invoke();
            
            stopwatch.Stop();
            RecordSuccess(grainType, methodName, stopwatch.ElapsedMilliseconds);
        }
        catch (Exception ex)
        {
            stopwatch.Stop();
            RecordFailure(grainType, methodName, stopwatch.ElapsedMilliseconds, ex);
            throw;
        }
    }
    
    private void RecordSuccess(string grainType, string method, double durationMs)
    {
        _grainCallDuration.Record(durationMs,
            new KeyValuePair<string, object?>("grain_type", grainType),
            new KeyValuePair<string, object?>("method", method),
            new KeyValuePair<string, object?>("status", "success"));
        
        _grainCallCount.Add(1,
            new KeyValuePair<string, object?>("grain_type", grainType),
            new KeyValuePair<string, object?>("method", method),
            new KeyValuePair<string, object?>("status", "success"));
        
        if (durationMs > 1000) // 1秒以上かかった場合は警告
        {
            _logger.LogWarning(
                "Slow grain call detected: {GrainType}.{Method} took {Duration}ms",
                grainType, method, durationMs);
        }
    }
}

よくあるパフォーマンス問題と解決策

1. Grain活性化の遅延

// 問題: OnActivateAsyncで重い処理
public class SlowActivationGrain : Grain
{
    public override async Task OnActivateAsync()
    {
        // ❌ 悪い例：活性化時に重い処理
        var data = await LoadLargeDataFromDatabaseAsync();
        ProcessComplexData(data);
    }
}

// 解決策: 遅延初期化パターン
public class OptimizedGrain : Grain
{
    private readonly Lazy<Task<ComplexData>> _lazyData;
    
    public OptimizedGrain()
    {
        _lazyData = new Lazy<Task<ComplexData>>(
            LoadDataAsync,
            LazyThreadSafetyMode.ExecutionAndPublication);
    }
    
    public override Task OnActivateAsync()
    {
        // ✅ 良い例：最小限の初期化のみ
        return Task.CompletedTask;
    }
    
    public async Task<Result> ProcessAsync()
    {
        // 必要な時に初めてデータをロード
        var data = await _lazyData.Value;
        return ProcessData(data);
    }
}

2. メモリリークの検出と修正

// メモリリーク検出Grain
public class MemoryMonitorGrain : Grain, IMemoryMonitorGrain
{
    private readonly List<WeakReference> _trackedObjects = new();
    private IDisposable _timer;
    
    public override Task OnActivateAsync()
    {
        _timer = RegisterTimer(
            CheckMemoryLeaks,
            null,
            TimeSpan.FromMinutes(1),
            TimeSpan.FromMinutes(1));
        
        return base.OnActivateAsync();
    }
    
    public Task TrackObjectAsync(object obj)
    {
        _trackedObjects.Add(new WeakReference(obj));
        return Task.CompletedTask;
    }
    
    private async Task CheckMemoryLeaks(object state)
    {
        // 強制的にGCを実行
        GC.Collect();
        GC.WaitForPendingFinalizers();
        GC.Collect();
        
        var leakedObjects = 0;
        foreach (var weakRef in _trackedObjects)
        {
            if (weakRef.IsAlive)
            {
                leakedObjects++;
            }
        }
        
        if (leakedObjects > 0)
        {
            GetLogger().LogWarning(
                "Potential memory leak detected: {LeakedObjects} objects still alive",
                leakedObjects);
        }
        
        // 死んだ参照をクリーンアップ
        _trackedObjects.RemoveAll(wr => !wr.IsAlive);
    }
}

// メモリリークを防ぐパターン
public class NoLeakGrain : Grain
{
    private readonly List<IDisposable> _disposables = new();
    private readonly CancellationTokenSource _cts = new();
    
    public override Task OnActivateAsync()
    {
        // イベントハンドラーの適切な管理
        var subscription = EventBus.Subscribe<MyEvent>(HandleEvent);
        _disposables.Add(subscription);
        
        // タイマーの適切な管理
        var timer = RegisterTimer(
            TimerCallback,
            null,
            TimeSpan.FromSeconds(30),
            TimeSpan.FromSeconds(30));
        _disposables.Add(timer);
        
        return base.OnActivateAsync();
    }
    
    public override async Task OnDeactivateAsync(DeactivationReason reason)
    {
        // リソースの確実な解放
        _cts.Cancel();
        
        foreach (var disposable in _disposables)
        {
            disposable?.Dispose();
        }
        _disposables.Clear();
        
        await base.OnDeactivateAsync(reason);
    }
}

3. デッドロックの回避

// デッドロックが発生しやすいパターン
public class DeadlockProneGrain : Grain
{
    // ❌ 悪い例：相互に呼び出し合う可能性
    public async Task<Result> MethodA(string otherGrainId)
    {
        var otherGrain = GrainFactory.GetGrain<IDeadlockProneGrain>(otherGrainId);
        return await otherGrain.MethodB(this.GetPrimaryKeyString());
    }
    
    public async Task<Result> MethodB(string callerGrainId)
    {
        var callerGrain = GrainFactory.GetGrain<IDeadlockProneGrain>(callerGrainId);
        return await callerGrain.MethodA(this.GetPrimaryKeyString());
    }
}

// デッドロックを回避するパターン
[Reentrant] // 再入可能にする
public class SafeGrain : Grain
{
    private readonly SemaphoreSlim _semaphore = new(1, 1);
    
    public async Task<Result> SafeMethodAsync()
    {
        // タイムアウト付きでロックを取得
        if (!await _semaphore.WaitAsync(TimeSpan.FromSeconds(5)))
        {
            throw new TimeoutException("Failed to acquire lock");
        }
        
        try
        {
            // 処理
            return await ProcessAsync();
        }
        finally
        {
            _semaphore.Release();
        }
    }
    
    // 非同期処理でのデッドロック回避
    public async Task<Result> AsyncSafeMethodAsync()
    {
        // ConfigureAwait(false)を使用
        var data = await LoadDataAsync().ConfigureAwait(false);
        
        // 同期的なブロッキングを避ける
        // ❌ 悪い: Task.Result や Task.Wait()
        // ✅ 良い: await を使用
        
        return ProcessData(data);
    }
}

4. ホットスポットの解消

// ホットスポットになりやすいGrain
public class HotspotGrain : Grain
{
    private int _counter;
    
    // ❌ 全てのリクエストが1つのGrainに集中
    public Task<int> IncrementGlobalCounterAsync()
    {
        return Task.FromResult(++_counter);
    }
}

// 負荷分散されたカウンター
public interface IDistributedCounterGrain : IGrainWithIntegerKey
{
    Task<long> IncrementAsync();
    Task<long> GetLocalCountAsync();
}

public interface ICounterAggregatorGrain : IGrainWithIntegerKey
{
    Task<long> GetTotalCountAsync();
}

public class DistributedCounterGrain : Grain, IDistributedCounterGrain
{
    private long _localCount;
    
    public Task<long> IncrementAsync()
    {
        return Task.FromResult(++_localCount);
    }
    
    public Task<long> GetLocalCountAsync()
    {
        return Task.FromResult(_localCount);
    }
}

public class CounterAggregatorGrain : Grain, ICounterAggregatorGrain
{
    private readonly int _shardCount = 100;
    
    public async Task<long> GetTotalCountAsync()
    {
        var tasks = Enumerable.Range(0, _shardCount)
            .Select(i => GrainFactory
                .GetGrain<IDistributedCounterGrain>(i)
                .GetLocalCountAsync())
            .ToList();
        
        var counts = await Task.WhenAll(tasks);
        return counts.Sum();
    }
}

// クライアント側の使用
public class CounterClient
{
    private readonly IGrainFactory _grainFactory;
    private readonly Random _random = new();
    
    public async Task<long> IncrementCounterAsync()
    {
        // ランダムなシャードに振り分け
        var shardId = _random.Next(100);
        var counter = _grainFactory.GetGrain<IDistributedCounterGrain>(shardId);
        return await counter.IncrementAsync();
    }
}

高度な最適化テクニック

1. バッチ処理の最適化

public interface IBatchProcessorGrain : IGrainWithGuidKey
{
    Task<BatchResult> ProcessBatchAsync(List<WorkItem> items);
}

public class OptimizedBatchProcessorGrain : Grain, IBatchProcessorGrain
{
    private readonly Channel<WorkItem> _workChannel;
    private readonly CancellationTokenSource _cts = new();
    private Task _processingTask;
    
    public OptimizedBatchProcessorGrain()
    {
        // バックプレッシャー付きチャネル
        _workChannel = Channel.CreateBounded<WorkItem>(new BoundedChannelOptions(1000)
        {
            FullMode = BoundedChannelFullMode.Wait,
            SingleReader = true,
            SingleWriter = false
        });
    }
    
    public override Task OnActivateAsync()
    {
        _processingTask = ProcessWorkItemsAsync(_cts.Token);
        return base.OnActivateAsync();
    }
    
    public async Task<BatchResult> ProcessBatchAsync(List<WorkItem> items)
    {
        var results = new List<Task<bool>>();
        
        foreach (var item in items)
        {
            // 非同期でチャネルに追加
            var tcs = new TaskCompletionSource<bool>();
            item.CompletionSource = tcs;
            
            await _workChannel.Writer.WriteAsync(item);
            results.Add(tcs.Task);
        }
        
        // 全ての処理完了を待機
        var completedResults = await Task.WhenAll(results);
        
        return new BatchResult
        {
            TotalItems = items.Count,
            SuccessCount = completedResults.Count(r => r),
            FailureCount = completedResults.Count(r => !r)
        };
    }
    
    private async Task ProcessWorkItemsAsync(CancellationToken cancellationToken)
    {
        var buffer = new List<WorkItem>(100);
        
        await foreach (var item in _workChannel.Reader.ReadAllAsync(cancellationToken))
        {
            buffer.Add(item);
            
            // バッファが満杯またはタイムアウトで処理
            if (buffer.Count >= 100 || 
                !_workChannel.Reader.TryRead(out _))
            {
                await ProcessBufferAsync(buffer);
                buffer.Clear();
            }
        }
    }
    
    private async Task ProcessBufferAsync(List<WorkItem> items)
    {
        try
        {
            // 効率的なバッチ処理
            await BulkProcessAsync(items);
            
            // 成功を通知
            foreach (var item in items)
            {
                item.CompletionSource?.SetResult(true);
            }
        }
        catch (Exception ex)
        {
            // 失敗を通知
            foreach (var item in items)
            {
                item.CompletionSource?.SetException(ex);
            }
        }
    }
}

2. メモリ効率の最適化

// メモリプールを使用した最適化
public class MemoryEfficientGrain : Grain
{
    private static readonly ArrayPool<byte> ByteArrayPool = ArrayPool<byte>.Shared;
    private static readonly ObjectPool<StringBuilder> StringBuilderPool = 
        new DefaultObjectPool<StringBuilder>(new StringBuilderPooledObjectPolicy());
    
    public async Task<ProcessResult> ProcessLargeDataAsync(Stream dataStream)
    {
        // ArrayPoolから配列を借用
        var buffer = ByteArrayPool.Rent(4096);
        
        try
        {
            var result = new ProcessResult();
            var bytesRead = 0;
            
            while ((bytesRead = await dataStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
            {
                // バッファを処理
                ProcessBuffer(buffer, bytesRead, result);
            }
            
            return result;
        }
        finally
        {
            // 配列を返却
            ByteArrayPool.Return(buffer, clearArray: true);
        }
    }
    
    public string BuildComplexString(List<DataItem> items)
    {
        // StringBuilderをプールから取得
        var sb = StringBuilderPool.Get();
        
        try
        {
            foreach (var item in items)
            {
                sb.AppendLine($"{item.Id}: {item.Name} - {item.Value}");
            }
            
            return sb.ToString();
        }
        finally
        {
            // StringBuilderを返却
            StringBuilderPool.Return(sb);
        }
    }
}

トラブルシューティングツール

デバッグ用Grain

public interface IDiagnosticGrain : IGrainWithIntegerKey
{
    Task<DiagnosticInfo> GetDiagnosticInfoAsync();
    Task<GrainStatistics> GetStatisticsAsync();
    Task ForceGarbageCollectionAsync();
}

public class DiagnosticGrain : Grain, IDiagnosticGrain
{
    public Task<DiagnosticInfo> GetDiagnosticInfoAsync()
    {
        var process = Process.GetCurrentProcess();
        
        return Task.FromResult(new DiagnosticInfo
        {
            // システム情報
            MachineName = Environment.MachineName,
            ProcessId = process.Id,
            ThreadCount = process.Threads.Count,
            
            // メモリ情報
            WorkingSet = process.WorkingSet64,
            PrivateMemory = process.PrivateMemorySize64,
            VirtualMemory = process.VirtualMemorySize64,
            GcHeapSize = GC.GetTotalMemory(false),
            Gen0Collections = GC.CollectionCount(0),
            Gen1Collections = GC.CollectionCount(1),
            Gen2Collections = GC.CollectionCount(2),
            
            // Orleans情報
            SiloAddress = RuntimeIdentity,
            GrainId = this.GetPrimaryKeyLong(),
            GrainType = this.GetType().Name,
            
            // カスタム診断情報
            CustomMetrics = CollectCustomMetrics()
        });
    }
    
    public async Task<GrainStatistics> GetStatisticsAsync()
    {
        // Grainの統計情報を収集
        var stats = new GrainStatistics();
        
        // リフレクションを使用して内部状態を取得
        var grainType = this.GetType();
        var fields = grainType.GetFields(BindingFlags.NonPublic | BindingFlags.Instance);
        
        foreach (var field in fields)
        {
            if (field.FieldType.IsGenericType && 
                field.FieldType.GetGenericTypeDefinition() == typeof(IPersistentState<>))
            {
                var state = field.GetValue(this);
                stats.StateSize = EstimateObjectSize(state);
            }
        }
        
        return stats;
    }
    
    public Task ForceGarbageCollectionAsync()
    {
        // 強制的にGCを実行（デバッグ用）
        GC.Collect();
        GC.WaitForPendingFinalizers();
        GC.Collect();
        
        return Task.CompletedTask;
    }
}

パフォーマンス監視ダッシュボード

// カスタムダッシュボードの実装
public class PerformanceDashboard : IHostedService
{
    private readonly IClusterClient _clusterClient;
    private readonly ILogger<PerformanceDashboard> _logger;
    private Timer _timer;
    
    public Task StartAsync(CancellationToken cancellationToken)
    {
        _timer = new Timer(
            CollectMetrics,
            null,
            TimeSpan.Zero,
            TimeSpan.FromSeconds(10));
        
        return Task.CompletedTask;
    }
    
    private async void CollectMetrics(object state)
    {
        try
        {
            // 各Siloから診断情報を収集
            var tasks = Enumerable.Range(0, 10)
                .Select(i => _clusterClient
                    .GetGrain<IDiagnosticGrain>(i)
                    .GetDiagnosticInfoAsync())
                .ToList();
            
            var diagnostics = await Task.WhenAll(tasks);
            
            // メトリクスを集計
            var aggregated = new AggregatedMetrics
            {
                TotalMemory = diagnostics.Sum(d => d.WorkingSet),
                AverageThreadCount = diagnostics.Average(d => d.ThreadCount),
                TotalGcCollections = diagnostics.Sum(d => 
                    d.Gen0Collections + d.Gen1Collections + d.Gen2Collections)
            };
            
            // ダッシュボードに送信
            await UpdateDashboardAsync(aggregated);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Failed to collect metrics");
        }
    }
}

まとめ

今回は、.NET Orleansのパフォーマンスチューニングとトラブルシューティングについて学びました。重要なポイント：

1. 計測の重要性: パフォーマンス改善は正確な測定から始まる
2. よくある問題: ホットスポット、メモリリーク、デッドロックへの対処
3. 最適化テクニック: バッチ処理、メモリプール、非同期パターン
4. 診断ツール: カスタム診断Grainとダッシュボード
5. 継続的な改善: 監視と最適化のサイクル

次回は、実際のプロダクション事例を通じて、Orleansを使った大規模システムの構築について解説します。

---

次回予告：「第6回：実践編 - 大規模リアルタイムシステムの構築事例」では、実際のプロジェクトでOrleansを使って構築した、100万同時接続を処理するシステムの設計と実装を詳しく解説します。